{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Importing Libraries","metadata":{}},{"cell_type":"code","source":"! pip install biosppy   ","metadata":{"execution":{"iopub.status.busy":"2021-07-28T07:44:07.032883Z","iopub.execute_input":"2021-07-28T07:44:07.033416Z","iopub.status.idle":"2021-07-28T07:44:19.058304Z","shell.execute_reply.started":"2021-07-28T07:44:07.033320Z","shell.execute_reply":"2021-07-28T07:44:19.057010Z"},"trusted":true},"execution_count":1,"outputs":[{"name":"stdout","text":"Collecting biosppy\n  Downloading biosppy-0.7.3.tar.gz (85 kB)\n\u001b[K     |████████████████████████████████| 85 kB 1.4 MB/s eta 0:00:011\n\u001b[?25hCollecting bidict\n  Downloading bidict-0.21.2-py2.py3-none-any.whl (37 kB)\nRequirement already satisfied: h5py in /opt/conda/lib/python3.7/site-packages (from biosppy) (2.10.0)\nRequirement already satisfied: matplotlib in /opt/conda/lib/python3.7/site-packages (from biosppy) (3.4.2)\nRequirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from biosppy) (1.19.5)\nRequirement already satisfied: scikit-learn in /opt/conda/lib/python3.7/site-packages (from biosppy) (0.23.2)\nRequirement already satisfied: scipy in /opt/conda/lib/python3.7/site-packages (from biosppy) (1.6.3)\nRequirement already satisfied: shortuuid in /opt/conda/lib/python3.7/site-packages (from biosppy) (1.0.1)\nRequirement already satisfied: six in /opt/conda/lib/python3.7/site-packages (from biosppy) (1.15.0)\nRequirement already satisfied: joblib in /opt/conda/lib/python3.7/site-packages (from biosppy) (1.0.1)\nRequirement already satisfied: opencv-python in /opt/conda/lib/python3.7/site-packages (from biosppy) (4.5.2.54)\nRequirement already satisfied: cycler>=0.10 in /opt/conda/lib/python3.7/site-packages (from matplotlib->biosppy) (0.10.0)\nRequirement already satisfied: pillow>=6.2.0 in /opt/conda/lib/python3.7/site-packages (from matplotlib->biosppy) (8.2.0)\nRequirement already satisfied: python-dateutil>=2.7 in /opt/conda/lib/python3.7/site-packages (from matplotlib->biosppy) (2.8.1)\nRequirement already satisfied: kiwisolver>=1.0.1 in /opt/conda/lib/python3.7/site-packages (from matplotlib->biosppy) (1.3.1)\nRequirement already satisfied: pyparsing>=2.2.1 in /opt/conda/lib/python3.7/site-packages (from matplotlib->biosppy) (2.4.7)\nRequirement already satisfied: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from scikit-learn->biosppy) (2.1.0)\nBuilding wheels for collected packages: biosppy\n  Building wheel for biosppy (setup.py) ... \u001b[?25ldone\n\u001b[?25h  Created wheel for biosppy: filename=biosppy-0.7.3-py2.py3-none-any.whl size=95409 sha256=b1a9a723ab44a3b37b1688a8905fd03c5ffcc12926abeca3b2c99c9d045408da\n  Stored in directory: /root/.cache/pip/wheels/2f/4f/8f/28b2adc462d7e37245507324f4817ce1c64ef2464f099f4f0b\nSuccessfully built biosppy\nInstalling collected packages: bidict, biosppy\nSuccessfully installed bidict-0.21.2 biosppy-0.7.3\n\u001b[33mWARNING: Running pip as root will break packages and permissions. You should install packages reliably by using venv: https://pip.pypa.io/warnings/venv\u001b[0m\n","output_type":"stream"}]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport wfdb\nimport os                                                                                                         \nimport gc\nimport scipy       \nimport sklearn\nfrom pathlib import Path\nfrom sklearn.utils import shuffle\nfrom sklearn.manifold import TSNE\nimport seaborn as sns\nfrom sklearn import preprocessing\nimport shutil\nimport math\nimport random\nfrom scipy.spatial import distance\nfrom biosppy.signals import ecg\nfrom scipy.interpolate import PchipInterpolator","metadata":{"execution":{"iopub.status.busy":"2021-07-28T07:44:19.060015Z","iopub.execute_input":"2021-07-28T07:44:19.060420Z","iopub.status.idle":"2021-07-28T07:44:26.664993Z","shell.execute_reply.started":"2021-07-28T07:44:19.060373Z","shell.execute_reply":"2021-07-28T07:44:26.663816Z"},"trusted":true},"execution_count":2,"outputs":[]},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection\n    print('Running on TPU ', tpu.cluster_spec().as_dict()['worker'])\nexcept ValueError:\n    raise BaseException('ERROR: Not connected to a TPU runtime; please see the previous cell in this notebook for instructions!')\n\ntf.config.experimental_connect_to_cluster(tpu)\ntf.tpu.experimental.initialize_tpu_system(tpu)\ntpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)","metadata":{"execution":{"iopub.status.busy":"2021-07-28T07:44:28.493027Z","iopub.execute_input":"2021-07-28T07:44:28.493448Z","iopub.status.idle":"2021-07-28T07:44:34.561725Z","shell.execute_reply.started":"2021-07-28T07:44:28.493409Z","shell.execute_reply":"2021-07-28T07:44:34.560660Z"},"trusted":true},"execution_count":3,"outputs":[{"name":"stdout","text":"Running on TPU  ['10.0.0.2:8470']\n","output_type":"stream"}]},{"cell_type":"markdown","source":"# Dataset Creation","metadata":{}},{"cell_type":"code","source":"####### Dataset Creation \n\n###### Constants\nFS = 500\nW_LEN = 256\nW_LEN_1_4 = 256 // 4\nW_LEN_3_4 = 3 * (256 // 4)\n\n###### Function to Read a Record\ndef read_rec(rec_path):\n\n    \"\"\" \n    Function to read record and return Segmented Signals\n\n    INPUTS:-\n    1) rec_path : Path of the Record\n\n    OUTPUTS:-\n    1) seg_sigs : Final Segmented Signals\n\n    \"\"\"\n    number_of_peaks = 2 # For extracting the required number of peaks                                    \n    full_rec = (wfdb.rdrecord(rec_path)).p_signal[:,1] # Entire Record\n\n    f = PchipInterpolator(np.arange(10000),full_rec) # Fitting Interpolation Function\n    x_samp = (np.arange(10000)*(500/360))[:7200] # Fixing Interpolation Input Values\n    full_rec_interp = f(x_samp)  # Intepolating Values \n    r_peaks_init = ecg.hamilton_segmenter(full_rec_interp,360)[0] # R-Peak Segmentation and input is the signal frequency of 500Hz in this case\n    final_peak_index = r_peaks_init[int(r_peaks_init.shape[0] - int((r_peaks_init.shape[0]%number_of_peaks)))-1]\n    r_peaks_final = r_peaks_init[:final_peak_index] # Final Number of R_Peaks\n    full_rec_final = full_rec_interp[:int(r_peaks_final[-1]+W_LEN)] # Final Sequence\n    seg_sigs, r_peaks_ref = segmentSignals(full_rec_final,list(r_peaks_final)) # Final Signal Segmentation\n\n    return seg_sigs # Returning the Ouput of the Signal Segmentation\n\n###### Function to Segment Signals\n\n##### Function\ndef segmentSignals(signal, r_peaks_annot, normalization=True, person_id= None, file_id=None):\n    \n    \"\"\"\n    Segments signals based on the detected R-Peak\n    Args:\n        signal (numpy array): input signal\n        r_peaks_annot (int []): r-peak locations.\n        normalization (bool, optional): apply z-normalization or not? . Defaults to True.\n        person_id ([type], optional): [description]. Defaults to None.\n        file_id ([type], optional): [description]. Defaults to None.\n    Returns:\n            [tuple(numpy array,numpy array)]: segmented signals and refined r-peaks\n    \"\"\"\n    def refine_rpeaks(signal, r_peaks):\n        \"\"\"\n        Refines the detected R-peaks. If the R-peak is slightly shifted, this assigns the \n        highest point R-peak.\n        Args:\n            signal (numpy array): input signal\n            r_peaks (int []): list of detected r-peaks\n        Returns:\n            [numpy array]: refined r-peaks\n        \"\"\"\n        r_peaks2 = np.array(r_peaks)            # make a copy\n        for i in range(len(r_peaks)):\n            r = r_peaks[i]          # current R-peak\n            small_segment = signal[max(0,r-100):min(len(signal),r+100)]         # consider the neighboring segment of R-peak\n            r_peaks2[i] = np.argmax(small_segment) - 100 + r_peaks[i]           # picking the highest point\n            r_peaks2[i] = min(r_peaks2[i],len(signal))                          # the detected R-peak shouldn't be outside the signal\n            r_peaks2[i] = max(r_peaks2[i],0)                                    # checking if it goes before zero    \n        return r_peaks2                     # returning the refined r-peak list\n    \n    segmented_signals = []                      # array containing the segmented beats\n    \n    r_peaks = np.array(r_peaks_annot)\n\n    r_peaks = refine_rpeaks(signal, r_peaks)\n    skip_len = 2 # Parameter to specify number of r_peaks in one signal\n    max_seq_len = 512 # Parameter to specify maximum sequence length\n    \n    for r_curr in range(0,int(r_peaks.shape[0]-(skip_len-1)),skip_len):\n        if ((r_peaks[r_curr]-W_LEN_1_4)<0) or ((r_peaks[r_curr+(skip_len-1)]+W_LEN_3_4)>=len(signal)):           # not enough signal to segment\n            continue\n        segmented_signal = np.array(signal[r_peaks[r_curr]-W_LEN_1_4:r_peaks[r_curr+(skip_len-1)]+W_LEN_3_4])        # segmenting a heartbeat\n        segmented_signal = list(segmented_signal)\n        #print(segmented_signal.shape)\n        \n        if(len(segmented_signal) < 512):\n            for m in range(int(512-len(segmented_signal))): # Zero Padding\n                segmented_signal.append(0)\n        else:\n            segmented_signal = (segmented_signal[:int(max_seq_len)])\n            \n        segmented_signal = np.array(segmented_signal)\n        \n        if(segmented_signal.shape != (512,1)):    \n            segmented_signal = np.reshape(segmented_signal,(512,1))\n            \n        if (normalization):             # Z-score normalization\n            if abs(np.std(segmented_signal))<1e-6:          # flat line ECG, will cause zero division error\n                continue\n            segmented_signal = (segmented_signal - np.mean(segmented_signal)) / np.std(segmented_signal)            \n              \n        #if not np.isnan(segmented_signal).any():                    # checking for nan, this will never happen\n            segmented_signals.append(segmented_signal)\n\n    return segmented_signals,r_peaks           # returning the segmented signals and the refined r-peaks","metadata":{"execution":{"iopub.status.busy":"2021-07-28T08:00:19.621542Z","iopub.execute_input":"2021-07-28T08:00:19.621980Z","iopub.status.idle":"2021-07-28T08:00:19.643958Z","shell.execute_reply.started":"2021-07-28T08:00:19.621947Z","shell.execute_reply":"2021-07-28T08:00:19.642444Z"},"trusted":true},"execution_count":11,"outputs":[]},{"cell_type":"code","source":"###### Numpy Array Creation - Training Dataset \npath_to_dir = '../input/ecg1d/ecg-id-database-1.0.0'\ntotal_folders = 90\ncurrent_index = 0\n\nX_train = []\nX_dev = []\ny_train = []\ny_dev = []\n\n#for item in subjects_with_two:\nfor i in range(2,74):\n\n    if(i != 75):\n\n        print(i-1)\n        folder_path = os.path.join(path_to_dir,np.sort(os.listdir(path_to_dir))[i]) # Path Selection\n        #items_in_folder = int(len(folder_path)//3)\n        #current_storage_path = './5_Beat_Ecg_ECG1D'+'/person'+str(current_index)\n\n        #for j in os.listdir(item):\n\n        for j in range(2):\n\n            rec_path = folder_path+'/'+'rec'+'_'+str(j+1) # Path to Record\n            seg_signal_current = read_rec(rec_path)\n\n            if(j == 0):\n                for k in range(len(seg_signal_current)):\n                    #file_name_current = current_storage_path+'/'+str(j)+'_/'+str(k)\n                    #np.savez_compressed(file_name_current,seg_signal_current[k])\n                    X_train.append(seg_signal_current[k])\n                    y_train.append(current_index)\n\n            else:\n                for k in range(len(seg_signal_current)):\n                    #file_name_current = current_storage_path+'/'+str(j)+'_/'+str(k)\n                    #np.savez_compressed(file_name_current,seg_signal_current[k])\n                    X_dev.append(seg_signal_current[k])\n                    y_dev.append(current_index)\n\n        current_index = current_index+1\n\n###### Shuffling Numpy Arrays\nX_train,y_train = shuffle(X_train,y_train)\nX_dev,y_dev = shuffle(X_dev,y_dev)","metadata":{"execution":{"iopub.status.busy":"2021-07-28T08:01:43.509205Z","iopub.execute_input":"2021-07-28T08:01:43.509618Z","iopub.status.idle":"2021-07-28T08:01:47.969360Z","shell.execute_reply.started":"2021-07-28T08:01:43.509572Z","shell.execute_reply":"2021-07-28T08:01:47.968216Z"},"trusted":true},"execution_count":15,"outputs":[{"name":"stdout","text":"1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n11\n12\n13\n14\n15\n16\n17\n18\n19\n20\n21\n22\n23\n24\n25\n26\n27\n28\n29\n30\n31\n32\n33\n34\n35\n36\n37\n38\n39\n40\n41\n42\n43\n44\n45\n46\n47\n48\n49\n50\n51\n52\n53\n54\n55\n56\n57\n58\n59\n60\n61\n62\n63\n64\n65\n66\n67\n68\n69\n70\n71\n72\n","output_type":"stream"}]},{"cell_type":"code","source":"###### Checking Shape of the Arrays\nprint(np.array(X_train).shape)\nprint(np.array(y_train).shape)\nprint(np.array(X_dev).shape)\nprint(np.array(y_dev).shape)\n\nprint(np.max(y_train))\nprint(np.min(y_dev))","metadata":{"execution":{"iopub.status.busy":"2021-07-28T08:01:50.776530Z","iopub.execute_input":"2021-07-28T08:01:50.776954Z","iopub.status.idle":"2021-07-28T08:01:50.790616Z","shell.execute_reply.started":"2021-07-28T08:01:50.776901Z","shell.execute_reply":"2021-07-28T08:01:50.789491Z"},"trusted":true},"execution_count":16,"outputs":[{"name":"stdout","text":"(865, 512, 1)\n(865,)\n(888, 512, 1)\n(888,)\n71\n0\n","output_type":"stream"}]},{"cell_type":"code","source":"###### Saving Numpy Arrays\nnp.savez_compressed('X_train_ECG1D_train_OSV.npz',np.array(X_train))\nnp.savez_compressed('y_train_ECG1D_train_OSV.npz',np.array(y_train))\nnp.savez_compressed('X_dev_ECG1D_train_OSV.npz',np.array(X_dev))\nnp.savez_compressed('y_dev_ECG1D_train_OSV.npz',np.array(y_dev))","metadata":{"execution":{"iopub.status.busy":"2021-07-28T08:02:29.547189Z","iopub.execute_input":"2021-07-28T08:02:29.547631Z","iopub.status.idle":"2021-07-28T08:02:29.905670Z","shell.execute_reply.started":"2021-07-28T08:02:29.547595Z","shell.execute_reply":"2021-07-28T08:02:29.904321Z"},"trusted":true},"execution_count":17,"outputs":[]},{"cell_type":"code","source":"###### Numpy Array Creation - Testing Dataset\npath_to_dir = '../input/ecg1d/ecg-id-database-1.0.0'\ntotal_folders = 90\ncurrent_index = 0\n\nX_train = []\nX_dev = []\ny_train = []\ny_dev = []\n\n#for item in subjects_with_two:\nfor i in range(74,92):\n\n    if(i != 75):\n\n        print(i-1)\n        folder_path = os.path.join(path_to_dir,np.sort(os.listdir(path_to_dir))[i]) # Path Selection\n        #items_in_folder = int(len(folder_path)//3)\n        #current_storage_path = './5_Beat_Ecg_ECG1D'+'/person'+str(current_index)\n\n        #for j in os.listdir(item):\n\n        for j in range(2):\n\n            rec_path = folder_path+'/'+'rec'+'_'+str(j+1) # Path to Record\n            seg_signal_current = read_rec(rec_path)\n\n            if(j == 0):\n                for k in range(len(seg_signal_current)):\n                    #file_name_current = current_storage_path+'/'+str(j)+'_/'+str(k)\n                    #np.savez_compressed(file_name_current,seg_signal_current[k])\n                    X_train.append(seg_signal_current[k])\n                    y_train.append(current_index)\n\n            else:\n                for k in range(len(seg_signal_current)):\n                    #file_name_current = current_storage_path+'/'+str(j)+'_/'+str(k)\n                    #np.savez_compressed(file_name_current,seg_signal_current[k])\n                    X_dev.append(seg_signal_current[k])\n                    y_dev.append(current_index)\n\n        current_index = current_index+1\n\n###### Shuffling Numpy Arrays\nX_train,y_train = shuffle(X_train,y_train)\nX_dev,y_dev = shuffle(X_dev,y_dev)\n\n###### Checking Shape of the Arrays\nprint(np.array(X_train).shape)\nprint(np.array(y_train).shape)\nprint(np.array(X_dev).shape)\nprint(np.array(y_dev).shape)\n\nprint(np.max(y_train))\nprint(np.min(y_dev))","metadata":{"execution":{"iopub.status.busy":"2021-07-28T08:32:39.851102Z","iopub.execute_input":"2021-07-28T08:32:39.851580Z","iopub.status.idle":"2021-07-28T08:32:41.408018Z","shell.execute_reply.started":"2021-07-28T08:32:39.851542Z","shell.execute_reply":"2021-07-28T08:32:41.407005Z"},"trusted":true},"execution_count":29,"outputs":[{"name":"stdout","text":"73\n75\n76\n77\n78\n79\n80\n81\n82\n83\n84\n85\n86\n87\n88\n89\n90\n(222, 512, 1)\n(222,)\n(222, 512, 1)\n(222,)\n16\n0\n","output_type":"stream"}]},{"cell_type":"code","source":"###### Saving Numpy Arrays\nnp.savez_compressed('X_train_ECG1D_test_OSV.npz',np.array(X_train))\nnp.savez_compressed('y_train_ECG1D_test_OSV.npz',np.array(y_train))\nnp.savez_compressed('X_dev_ECG1D_test_OSV.npz',np.array(X_dev))\nnp.savez_compressed('y_dev_ECG1D_test_OSV.npz',np.array(y_dev))  ","metadata":{"execution":{"iopub.status.busy":"2021-07-28T08:41:02.628359Z","iopub.execute_input":"2021-07-28T08:41:02.628955Z","iopub.status.idle":"2021-07-28T08:41:02.730520Z","shell.execute_reply.started":"2021-07-28T08:41:02.628900Z","shell.execute_reply":"2021-07-28T08:41:02.729381Z"},"trusted":true},"execution_count":30,"outputs":[]},{"cell_type":"code","source":"##### Loading Dataset\nX_train = np.array(np.load('./X_train_ECG1D_test_OSV.npz',allow_pickle=True)['arr_0'],dtype=np.float16)\nX_dev = np.array(np.load('./X_dev_ECG1D_test_OSV.npz',allow_pickle=True)['arr_0'],dtype=np.float16)\ny_train = np.load('./y_train_ECG1D_test_OSV.npz',allow_pickle=True)['arr_0']\ny_dev = np.load('./y_dev_ECG1D_test_OSV.npz',allow_pickle=True)['arr_0']","metadata":{"execution":{"iopub.status.busy":"2021-07-28T08:41:28.205190Z","iopub.execute_input":"2021-07-28T08:41:28.205627Z","iopub.status.idle":"2021-07-28T08:41:28.234053Z","shell.execute_reply.started":"2021-07-28T08:41:28.205586Z","shell.execute_reply":"2021-07-28T08:41:28.232948Z"},"trusted":true},"execution_count":31,"outputs":[]},{"cell_type":"code","source":"##### Converting Labels to Categorical Format\ny_train_ohot = tf.keras.utils.to_categorical(y_train)\ny_dev_ohot = tf.keras.utils.to_categorical(y_dev)","metadata":{"execution":{"iopub.status.busy":"2021-07-28T08:41:30.255987Z","iopub.execute_input":"2021-07-28T08:41:30.256383Z","iopub.status.idle":"2021-07-28T08:41:30.261852Z","shell.execute_reply.started":"2021-07-28T08:41:30.256352Z","shell.execute_reply":"2021-07-28T08:41:30.260599Z"},"trusted":true},"execution_count":32,"outputs":[]},{"cell_type":"markdown","source":"# Model Making","metadata":{}},{"cell_type":"markdown","source":"## Self-Calibrated Convolution","metadata":{}},{"cell_type":"code","source":"###### Model Development : Self-Calibrated \n\n##### Defining Self-Calibrated Block\n\nrate_regularizer = 1e-5\nclass self_cal_Conv1D(tf.keras.layers.Layer):\n\n    \"\"\" \n    This is inherited class from keras.layers and shall be instatition of self-calibrated convolutions\n    \"\"\"\n    \n    def __init__(self,num_filters,kernel_size,num_features):\n    \n        #### Defining Essentials\n        super().__init__()\n        self.num_filters = num_filters\n        self.kernel_size = kernel_size\n        self.num_features = num_features # Number of Channels in Input\n\n        #### Defining Layers\n        self.conv2 = tf.keras.layers.Conv1D(self.num_features/2,self.kernel_size,padding='same',kernel_regularizer=tf.keras.regularizers.l2(rate_regularizer),dtype='float32',activation='relu')\n        self.conv3 = tf.keras.layers.Conv1D(self.num_features/2,self.kernel_size,padding='same',kernel_regularizer=tf.keras.regularizers.l2(rate_regularizer),dtype='float32',activation='relu')\n        self.conv4 = tf.keras.layers.Conv1D(self.num_filters/2,self.kernel_size,padding='same',activation='relu',kernel_regularizer=tf.keras.regularizers.l2(rate_regularizer),dtype='float32')\n        self.conv1 = tf.keras.layers.Conv1D(self.num_filters/2,self.kernel_size,padding='same',activation='relu',kernel_regularizer=tf.keras.regularizers.l2(rate_regularizer),dtype='float32')\n        self.upsample = tf.keras.layers.Conv1DTranspose(filters=int(self.num_features/2),kernel_size=4,strides=4)\n        #self.attention_layer = tf.keras.layers.Attention()\n        #self.lstm = tf.keras.layers.LSTM(int(self.num_features/2),return_sequences=True)\n        #self.layernorm = tf.keras.layers.LayerNormalization()\n    \n    def get_config(self):\n\n        config = super().get_config().copy()\n        config.update({\n            'num_filters': self.num_filters,\n            'kernel_size': self.kernel_size,\n            'num_features': self.num_features\n        })\n        return config\n    \n    \n    def call(self,X):\n       \n        \"\"\"\n          INPUTS : 1) X - Input Tensor of shape (batch_size,sequence_length,num_features)\n          OUTPUTS : 1) X - Output Tensor of shape (batch_size,sequence_length,num_features)\n        \"\"\"\n        \n        #### Dimension Extraction\n        b_s = (X.shape)[0] \n        seq_len = (X.shape)[1]\n        num_features = (X.shape)[2]\n        \n        #### Channel-Wise Division\n        X_attention = X[:,:,0:int(self.num_features/2)]\n        X_global = X[:,:,int(self.num_features/2):]\n        \n        #### Self Calibration Block\n\n        ### Local Feature Detection\n\n        ## Down-Sampling\n        #x1 = X_attention[:,0:int(seq_len/5),:]\n        #x2 = X_attention[:,int(seq_len/5):int(seq_len*(2/5)),:]\n        #x3 = X_attention[:,int(seq_len*(2/5)):int(seq_len*(3/5)),:]\n        #x4 = X_attention[:,int(seq_len*(3/5)):int(seq_len*(4/5)),:]\n        #x5 = X_attention[:,int(seq_len*(4/5)):seq_len,:]\n        x_down_sampled = tf.keras.layers.AveragePooling1D(pool_size=4,strides=4)(X_attention)\n        \n        ## Convoluting Down Sampled Sequence \n        #x1 = self.conv2(x1)\n        #x2 = self.conv2(x2)\n        #x3 = self.conv2(x3)\n        #x4 = self.conv2(x4)\n        #x5 = self.conv2(x5)\n        x_down_conv = self.conv2(x_down_sampled)\n        #x_down_feature = self.attention_layer([x_down_sampled,x_down_sampled])\n        #x_down_feature = self.lstm(x_down_sampled)\n        #x_down_feature = self.layernorm(x_down_feature)\n        \n        ## Up-Sampling\n        x_down_upsampled = self.upsample(x_down_conv)   \n        #X_local_upsampled = tf.keras.layers.concatenate([x1,x2,x3,x4,x5],axis=1)\n\n        ## Local-CAM\n        X_local = X_attention + x_down_upsampled  #X_local_upsampled\n\n        ## Local Importance \n        X_2 = tf.keras.activations.sigmoid(X_local)\n\n        ### Self-Calibration\n\n        ## Global Convolution\n        X_3 = self.conv3(X_attention)\n\n        ## Attention Determination\n        X_attention = tf.math.multiply(X_2,X_3)\n\n        #### Self-Calibration Feature Extraction\n        X_4 = self.conv4(X_attention)\n\n        #### Normal Feature Extraction\n        X_1 = self.conv1(X_global)\n\n        #### Concatenating and Returning Output\n        return (tf.keras.layers.concatenate([X_1,X_4],axis=2))","metadata":{"execution":{"iopub.status.busy":"2021-07-28T08:03:45.469922Z","iopub.execute_input":"2021-07-28T08:03:45.470632Z","iopub.status.idle":"2021-07-28T08:03:45.488097Z","shell.execute_reply.started":"2021-07-28T08:03:45.470594Z","shell.execute_reply":"2021-07-28T08:03:45.486918Z"},"trusted":true},"execution_count":21,"outputs":[]},{"cell_type":"markdown","source":"## Transformer","metadata":{}},{"cell_type":"code","source":"def get_angles(pos, i, d_model):\n    angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model))\n    return pos * angle_rates\n\ndef positional_encoding(position, d_model):\n    angle_rads = get_angles(np.arange(position)[:, np.newaxis],\n                          np.arange(d_model)[np.newaxis, :],\n                          d_model)\n  \n  # apply sin to even indices in the array; 2i\n    angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])\n  \n  # apply cos to odd indices in the array; 2i+1\n    angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])\n    \n    pos_encoding = angle_rads[np.newaxis, ...]\n    \n    return tf.cast(pos_encoding, dtype=tf.float32)\n\ndef create_padding_mask(seq):\n    seq = tf.cast(tf.math.equal(seq, 0), tf.float32)\n  \n    # add extra dimensions to add the padding\n    # to the attention logits. \n    return seq[:, tf.newaxis, tf.newaxis, :]  # (batch_size, 1, 1, seq_len)\n\ndef scaled_dot_product_attention(q, k, v, mask):\n    \"\"\"Calculate the attention weights.\n    q, k, v must have matching leading dimensions.\n    k, v must have matching penultimate dimension, i.e.: seq_len_k = seq_len_v.\n    The mask has different shapes depending on its type(padding or look ahead) \n    but it must be broadcastable for addition.\n\n    Args:\n    q: query shape == (..., seq_len_q, depth)\n    k: key shape == (..., seq_len_k, depth)\n    v: value shape == (..., seq_len_v, depth_v)\n    mask: Float tensor with shape broadcastable \n          to (..., seq_len_q, seq_len_k). Defaults to None.\n\n    Returns:\n    output, attention_weights\n    \"\"\"\n\n    matmul_qk = tf.matmul(q, k, transpose_b=True)  # (..., seq_len_q, seq_len_k)\n  \n    # scale matmul_qk\n    dk = tf.cast(tf.shape(k)[-1], tf.float32)\n    scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)\n\n    # add the mask to the scaled tensor.\n    if mask is not None:\n        scaled_attention_logits += (mask * -1e9)  \n\n    # softmax is normalized on the last axis (seq_len_k) so that the scores\n    # add up to 1.\n    attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)  # (..., seq_len_q, seq_len_k)\n\n    output = tf.matmul(attention_weights, v)  # (..., seq_len_q, depth_v)\n\n    return output, attention_weights\n\nclass MultiHeadAttention(tf.keras.layers.Layer):\n    def __init__(self, d_model, num_heads):\n        super(MultiHeadAttention, self).__init__()\n        self.num_heads = num_heads\n        self.d_model = d_model\n\n        assert d_model % self.num_heads == 0\n\n        self.depth = d_model // self.num_heads\n\n        self.wq = tf.keras.layers.Dense(d_model)\n        self.wk = tf.keras.layers.Dense(d_model)\n        self.wv = tf.keras.layers.Dense(d_model)\n\n        self.dense = tf.keras.layers.Dense(d_model)\n\n    def get_config(self):\n        config = super(MultiHeadAttention, self).get_config().copy()\n        config.update({\n            'd_model': self.d_model,\n            'num_heads':self.num_heads\n        })\n        \n    def split_heads(self, x, batch_size):\n        \n        \"\"\"Split the last dimension into (num_heads, depth).\n        Transpose the result such that the shape is (batch_size, num_heads, seq_len, depth)\n        \"\"\"\n        x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))\n        return tf.transpose(x, perm=[0, 2, 1, 3])\n    \n    def call(self, v, k, q, mask):\n        batch_size = tf.shape(q)[0]\n\n        q = self.wq(q)  # (batch_size, seq_len, d_model)\n        k = self.wk(k)  # (batch_size, seq_len, d_model)\n        v = self.wv(v)  # (batch_size, seq_len, d_model)\n\n        q = self.split_heads(q, batch_size)  # (batch_size, num_heads, seq_len_q, depth)\n        k = self.split_heads(k, batch_size)  # (batch_size, num_heads, seq_len_k, depth)\n        v = self.split_heads(v, batch_size)  # (batch_size, num_heads, seq_len_v, depth)\n\n        # scaled_attention.shape == (batch_size, num_heads, seq_len_q, depth)\n        # attention_weights.shape == (batch_size, num_heads, seq_len_q, seq_len_k)\n        scaled_attention, attention_weights = scaled_dot_product_attention(\n            q, k, v, mask)\n\n        scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])  # (batch_size, seq_len_q, num_heads, depth)\n\n        concat_attention = tf.reshape(scaled_attention, \n                                      (batch_size, -1, self.d_model))  # (batch_size, seq_len_q, d_model)\n\n        output = self.dense(concat_attention)  # (batch_size, seq_len_q, d_model)\n\n        return output, attention_weights\n\ndef point_wise_feed_forward_network(d_model, dff):\n    return tf.keras.Sequential([\n      tf.keras.layers.Dense(dff, activation='relu'),  # (batch_size, seq_len, dff)\n      tf.keras.layers.Dense(d_model)  # (batch_size, seq_len, d_model)\n  ])\n\nclass Encoder(tf.keras.layers.Layer):\n    def __init__(self, num_layers, d_model, num_heads, dff,\n               maximum_position_encoding, rate=0.1):\n        super(Encoder, self).__init__()\n\n        self.d_model = d_model\n        self.num_layers = num_layers\n        self.num_heads = num_heads\n        self.dff = dff\n        self.maximum_position_encoding = maximum_position_encoding\n        self.rate = rate\n\n        #self.embedding = tf.keras.layers.Embedding(input_vocab_size, d_model)\n        self.pos_encoding = positional_encoding(maximum_position_encoding, \n                                                self.d_model)\n\n\n        self.enc_layers = [EncoderLayer(d_model, num_heads, dff, rate) \n                           for _ in range(num_layers)]\n\n        self.dropout = tf.keras.layers.Dropout(rate)\n        \n    def get_config(self):\n        config = super(Encoder, self).get_config().copy()\n        config.update({\n            'num_layers': self.num_layers,\n            'd_model': self.d_model,\n            'num_heads':self.num_heads,\n            'dff':self.dff,\n            'maximum_position_encoding':self.maximum_position_encoding,\n            'rate':self.rate  \n        })\n        \n    def call(self, x, training, mask):\n\n        seq_len = tf.shape(x)[1]\n\n        # adding embedding and position encoding.\n        #x = self.embedding(x)  # (batch_size, input_seq_len, d_model)\n        x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n        x += self.pos_encoding[:, :seq_len, :]\n\n        x = self.dropout(x, training=training)         \n\n        for i in range(self.num_layers):\n            x = self.enc_layers[i](x, training, mask)\n\n        return x  # (batch_size, input_seq_len, d_model)\n\nclass EncoderLayer(tf.keras.layers.Layer):\n    def __init__(self, d_model, num_heads, dff, rate=0.1):\n        super(EncoderLayer, self).__init__()\n        \n        self.d_model = d_model\n        self.num_heads = num_heads\n        self.dff = dff\n        self.rate = rate\n\n        self.mha = MultiHeadAttention(d_model, num_heads)\n        self.ffn = point_wise_feed_forward_network(d_model, dff)\n\n        self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n\n        self.dropout1 = tf.keras.layers.Dropout(rate)\n        self.dropout2 = tf.keras.layers.Dropout(rate)\n        \n    def get_config(self):\n        config = super(EncoderLayer, self).get_config().copy()\n        config.update({\n            'd_model': self.d_model,\n            'num_heads':self.num_heads,\n            'dff':self.dff,\n            'rate':self.rate  \n        })\n\n    def call(self, x, training, mask):\n\n        attn_output, _ = self.mha(x, x, x, mask)  # (batch_size, input_seq_len, d_model)\n        attn_output = self.dropout1(attn_output, training=training)\n        out1 = self.layernorm1(x + attn_output)  # (batch_size, input_seq_len, d_model)\n\n        ffn_output = self.ffn(out1)  # (batch_size, input_seq_len, d_model)\n        ffn_output = self.dropout2(ffn_output, training=training)\n        out2 = self.layernorm2(out1 + ffn_output)  # (batch_size, input_seq_len, d_model)\n    \n        return out2\n    \nclass Transformer(tf.keras.Model):\n    def __init__(self, num_layers, d_model, num_heads, dff, \n                 pe_input, rate=0.1):\n        super(Transformer, self).__init__()\n        \n        self.num_layers = num_layers\n        self.d_model = d_model\n        self.num_heads = num_heads\n        self.dff = dff\n        self.pe_input = pe_input\n        self.rate = rate\n        \n        self.encoder = Encoder(num_layers, d_model, num_heads, dff, \n                                pe_input, rate)\n        \n    def get_config(self):\n        config = super(Transformer,self).get_config().copy()\n        config.update({\n            'num_layers': self.num_layers,\n            'd_model': self.d_model,\n            'num_heads':self.num_heads,\n            'dff':self.dff,\n            'pe_input':self.pe_input,\n            'rate':self.rate  \n        })\n    \n    def call(self, inp, training, enc_padding_mask):\n        return self.encoder(inp, training, enc_padding_mask)     ","metadata":{"execution":{"iopub.status.busy":"2021-07-28T08:04:09.201596Z","iopub.execute_input":"2021-07-28T08:04:09.202057Z","iopub.status.idle":"2021-07-28T08:04:09.246330Z","shell.execute_reply.started":"2021-07-28T08:04:09.202015Z","shell.execute_reply":"2021-07-28T08:04:09.245225Z"},"trusted":true},"execution_count":23,"outputs":[]},{"cell_type":"markdown","source":"## ArcFace Loss","metadata":{}},{"cell_type":"code","source":"class ArcFace(tf.keras.layers.Layer):\n    \n    def __init__(self, n_classes, s, m,regularizer):\n        super().__init__()\n        self.n_classes = n_classes\n        self.s = s\n        self.m = m\n        self.regularizer = tf.keras.regularizers.get(regularizer)\n\n    def get_config(self):\n\n        config = super().get_config().copy()\n        config.update({\n            'n_classes': self.n_classes,\n            's': self.s,\n            'm': self.m,\n            'regularizer': self.regularizer\n        })\n        return config\n\n    def build(self, input_shape):\n        super(ArcFace, self).build(input_shape[0])\n        self.W = self.add_weight(name='W',\n                                shape=(input_shape[0][-1], self.n_classes),\n                                initializer='glorot_uniform',\n                                trainable=True\n                                )\n\n    def call(self, inputs):\n        x, y = inputs\n        c = tf.keras.backend.shape(x)[-1]\n        # normalize feature\n        x = tf.nn.l2_normalize(x, axis=1)\n        # normalize weights\n        W = tf.nn.l2_normalize(self.W, axis=0)\n        # dot product\n        logits = x @ W\n        # add margin\n        # clip logits to prevent zero division when backward\n        theta = tf.acos(tf.keras.backend.clip(logits, -1.0 + tf.keras.backend.epsilon(), 1.0 - tf.keras.backend.epsilon()))\n        target_logits = tf.cos(theta + self.m)\n        # sin = tf.sqrt(1 - logits**2)\n        # cos_m = tf.cos(logits)\n        # sin_m = tf.sin(logits)\n        # target_logits = logits * cos_m - sin * sin_m\n        #\n        logits = logits * (1 - y) + target_logits * y\n        # feature re-scale\n        logits *= self.s\n        out = tf.nn.softmax(logits)    \n        return out\n\n    def compute_output_shape(self, input_shape):\n        return (None, self.n_classes)","metadata":{"execution":{"iopub.status.busy":"2021-07-28T08:04:12.213257Z","iopub.execute_input":"2021-07-28T08:04:12.213661Z","iopub.status.idle":"2021-07-28T08:04:12.226082Z","shell.execute_reply.started":"2021-07-28T08:04:12.213608Z","shell.execute_reply":"2021-07-28T08:04:12.224977Z"},"trusted":true},"execution_count":24,"outputs":[]},{"cell_type":"markdown","source":"# Model Training","metadata":{}},{"cell_type":"code","source":"####### Phase-1 Models\n###### Defining Architecture\n\nwith tpu_strategy.scope():\n\n    ##### SC_Module \n\n    #### Defining Hyperparameters\n    num_layers = 2\n    d_model = 512\n    num_heads = 8\n    dff = 1024\n    max_seq_len = 512 #X_train.shape[1]\n    pe_input = 128\n    rate = 0.5\n    num_features = 1\n    num_classes = 17\n\n    #### Defining Layers\n    Input_layer = tf.keras.layers.Input(shape=(max_seq_len,num_features))\n    self_conv1 = self_cal_Conv1D(128,15,128)\n    self_conv2 = self_cal_Conv1D(128,20,128) # Newly Added\n    self_conv3 = self_cal_Conv1D(256,15,128)\n    self_conv4 = self_cal_Conv1D(256,20,256) # Newly Added\n    self_conv5 = self_cal_Conv1D(512,15,256)\n    self_conv6 = self_cal_Conv1D(512,20,512) # Newly Added\n    self_conv7 = self_cal_Conv1D(1024,3,512)\n    self_conv8 = self_cal_Conv1D(1024,5,1024) # Newly Added\n    conv_initial = tf.keras.layers.Conv1D(32,15,padding='same',activation='relu')\n    conv_second = tf.keras.layers.Conv1D(64,15,padding='same',activation='relu')\n    conv_third = tf.keras.layers.Conv1D(128,15,padding='same',activation='relu')\n    #lstm1 = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(128,activation='tanh',return_sequences=True),merge_mode='ave')\n    transform_1 = tf.keras.layers.Conv1D(128,3,padding='same',kernel_initializer='lecun_normal', activation='selu')\n    transform_2 = tf.keras.layers.Conv1D(256,3,padding='same',kernel_initializer='lecun_normal', activation='selu')\n    transform_3 = tf.keras.layers.Conv1D(512,3,padding='same',kernel_initializer='lecun_normal', activation='selu')\n    transform_4 = tf.keras.layers.Conv1D(1024,3,padding='same',kernel_initializer='lecun_normal', activation='selu')\n    transformer = Transformer(num_layers,d_model,num_heads,dff,pe_input,rate)\n    gap_layer = tf.keras.layers.GlobalAveragePooling1D()\n    arc_logit_layer = ArcFace(17,30.0,0.3,tf.keras.regularizers.l2(1e-4))\n\n    #### Defining Architecture\n    ### Input Layer\n    Inputs = Input_layer\n    Input_Labels = tf.keras.layers.Input(shape=(num_classes,))\n\n    ### Initial Convolutional Layers\n    conv_initial = conv_initial(Inputs)\n    #conv_initial = tf.keras.layers.LayerNormalization()(conv_initial)\n    #conv_initial = tf.keras.layers.MaxPool1D(pool_size=2,strides=2)(conv_initial)     \n    #conv_initial = tf.keras.layers.Add()([conv_initial,Inputs])\n    \n    conv_second = conv_second(conv_initial)\n    #conv_second = tf.keras.layers.LayerNormalization()(conv_second)\n    #conv_second = tf.keras.layers.MaxPool1D(pool_size=2,strides=2)(conv_second)\n    #conv_second = tf.keras.layers.Add()([conv_second,conv_initial])\n    #conv_second = tf.keras.layers.concatenate(axis=2)([conv_initial,conv_second])\n    \n    conv_third = conv_third(conv_second)\n    #conv_third = tf.keras.layers.LayerNormalization()(conv_third)\n    #conv_third = tf.keras.layers.MaxPool1D(pool_size=2,strides=2)(conv_third)\n    #mask = tf.keras.layers.MaxPool1D(pool_size=2,strides=2)(Inputs)\n    #conv_third = tf.keras.layers.Add()([conv_third,conv_second])\n    #conv_third = tf.keras.layers.concatenate(axis=2)([conv_initial,conv_second,conv_third])\n    #conv_third = lstm1(conv_second)\n    #conv_third = tf.keras.layers.Attention()([conv_third,conv_third])\n    \n    ### 1st Residual Block\n    transform_1 = transform_1(conv_third)\n    conv1 = self_conv1(conv_third)\n    #conv1 = tf.keras.layers.AlphaDropout(rate=0.2)(conv1)\n    conv2 = self_conv2(conv1)\n    #conv2 = tf.keras.layers.AlphaDropout(rate=0.2)(conv2)\n    conv2 = tf.keras.layers.Add()([conv2,transform_1])\n    #conv2 = tf.keras.layers.LayerNormalization()(conv2)\n    conv2 = tf.keras.layers.MaxPool1D(pool_size=2,strides=2)(conv2)\n    #mask = tf.keras.layers.MaxPool1D(pool_size=2,strides=2)(mask)    \n\n    ### 2nd Residual Block\n    #conv_third = tf.keras.layers.Attention()([conv_third,conv_third])\n    transform_2 = transform_2(conv2)\n    conv3 = self_conv3(conv2)\n    #conv3 = tf.keras.layers.AlphaDropout(rate=0.2)(conv3)\n    conv4 = self_conv4(conv3)\n    #conv4 = tf.keras.layers.AlphaDropout(rate=0.2)(conv4)\n    conv4 = tf.keras.layers.Add()([conv4,transform_2])\n    #conv4 = tf.keras.layers.LayerNormalization()(conv4)\n    conv4 = tf.keras.layers.MaxPool1D(pool_size=2,strides=2)(conv4)\n    #mask = tf.keras.layers.MaxPool1D(pool_size=2,strides=2)(mask)\n\n    ### 3rd Residual Block\n    transform_3 = transform_3(conv4)\n    conv5 = self_conv5(conv4)\n    #conv5 = tf.keras.layers.AlphaDropout(rate=0.2)(conv5)\n    conv6 = self_conv6(conv5)\n    #conv6 = tf.keras.layers.AlphaDropout(rate=0.2)(conv6)\n    conv6 = tf.keras.layers.Add()([conv6,transform_3])\n    #conv6 = tf.keras.layers.LayerNormalization()(conv6)\n    #conv6 = tf.keras.layers.MaxPool1D(pool_size=2,strides=2)(conv6)\n\n    ### 4th Residual Block\n    #transform_4 = transform_4(conv6)\n    #conv7 = self_conv7(conv6)\n    #conv8 = self_conv8(conv7)\n    #conv8 = tf.keras.layers.Add()([conv8,transform_4])\n\n    ### Transformer\n    ## Wide-Head Attention Model\n    #tx_embedding = tf.keras.layers.Lambda(PE_Layer)(Inputs)\n    #tx_embedding = tf.keras.layers.Dropout(rate)(tx_embedding,training=True)\n    #mask_reshaped = tf.keras.layers.Reshape((max_seq_len,))(Inputs)\n    #encoder_op1 = encoder_block1(tx_embedding,mask_reshaped)\n    #encoder_op2 = encoder_block2(encoder_op1,mask_reshaped)\n\n    ## Narrow-Head Attention Model\n    #mask_reshaped = tf.keras.layers.Reshape((160,))(mask)\n    embeddings =  transformer(inp=conv6,enc_padding_mask=None)\n    #embeddings = transformer(inp=conv6,enc_padding_mask=create_padding_mask(mask))\n    #residual_embeddings = tf.keras.layers.Add()([conv6,embeddings])\n\n    ### Output Layers\n    ## Initial Layers\n    gap_op = gap_layer(embeddings)\n    dense1 = tf.keras.layers.Dense(256,activation='relu')(gap_op)\n    dropout1 = tf.keras.layers.Dropout(rate)(dense1)\n    \n    ## ArcFace Output Network\n    dense2 = tf.keras.layers.Dense(256,kernel_initializer='he_normal',\n                kernel_regularizer=tf.keras.regularizers.l2(1e-4))(dropout1)\n    ##dense2 = tf.keras.layers.BatchNormalization()(dense2)\n    dense3 = arc_logit_layer(([dense2,Input_Labels]))\n    \n    ## Softmax Output Network\n    #dense2 = tf.keras.layers.Dense(256,activation='relu')(dropout1)\n    ###dropout2 = tf.keras.layers.Dropout(rate)(dense2) # Not to be included\n    #dense3 = tf.keras.layers.Dense(35,activation='softmax')(dense2)\n\n    #### Compiling Architecture            \n    ### ArcFace Model Compilation\n    model = tf.keras.models.Model(inputs=[Inputs,Input_Labels],outputs=dense3)\n    ### Softmax Model Compilation\n    #model = tf.keras.models.Model(inputs=Inputs,outputs=dense3)\n    model.load_weights('./Identification_ECG1D_5.h5')\n    model.compile(tf.keras.optimizers.Adam(lr=1e-4,clipnorm=1.0),loss='categorical_crossentropy',metrics=['accuracy'])\n\nmodel.summary()      \ntf.keras.utils.plot_model(model)\n##### Model Training \n\n#### Model Checkpointing\nfilepath = './Identification_ECG1D_5.h5'\ncheckpoint = tf.keras.callbacks.ModelCheckpoint(filepath,monitor='val_accuracy',save_best_only=True,mode='max',save_weights_only=True)\n\n#### Custom Learning Rate Schedule\n#def build_lrfn(lr_start=1e-4, lr_max=1e-3, \n#               lr_min=1e-6, lr_rampup_epochs=5, \n#               lr_sustain_epochs=0, lr_exp_decay=.87):\n#    lr_max = lr_max * tpu_strategy.num_replicas_in_sync\n\n#    def lrfn(epoch):\n#        if epoch < lr_rampup_epochs:\n#            lr = (lr_max - lr_start) / lr_rampup_epochs * epoch + lr_start\n#        elif epoch < lr_rampup_epochs + lr_sustain_epochs:\n#            lr = lr_max\n#        else:\n#            lr = (lr_max - lr_min) * lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs) + lr_min#\n#        return lr\n#    \n#    return lrfn\n\n#lrfn = build_lrfn()\n#lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=1)\n#callback_list = [checkpoint,  lr_callback]\n\n#### Model Training\n#### Model Training\n### ArcFace Training\n#history = model.fit((X_train,y_train_ohot),y_train_ohot,epochs=500,batch_size=128,\n#                validation_data=((X_dev,y_dev_ohot),y_dev_ohot),validation_batch_size=128,\n#               callbacks=checkpoint)\n\n### Softmax Training \n#history = model.fit(X_train,y_train_ohot,epochs=250,batch_size=128,\n#                validation_data=(X_dev,y_dev_ohot),validation_batch_size=128,\n#                callbacks=checkpoint)\n\n\n##### Plotting Metrics  \n#### Accuracy and Loss Plots \n\n### Accuracy\n#plt.plot(history.history['accuracy'])\n#plt.plot(history.history['val_accuracy'])\n#plt.title('Model Accuracy')\n#plt.ylabel('Accuracy')\n#plt.xlabel('Epoch')  \n#plt.legend(['Train', 'Validation'], loc='best')\n#plt.show()\n\n### Loss     \n#plt.plot(history.history['loss'])  \n#plt.plot(history.history['val_loss'])\n#plt.title('Model Loss')  \n#plt.ylabel('Loss')         \n#plt.xlabel('epoch')\n#plt.legend(['Train', 'Validation'], loc='best')   \n#plt.show()\n\n##### Saving Model            \n#model.save_weights('ECG_SCNRNet.h5' ","metadata":{"execution":{"iopub.status.busy":"2021-07-28T08:41:44.545501Z","iopub.execute_input":"2021-07-28T08:41:44.546087Z","iopub.status.idle":"2021-07-28T08:41:48.574579Z","shell.execute_reply.started":"2021-07-28T08:41:44.546050Z","shell.execute_reply":"2021-07-28T08:41:48.573357Z"},"trusted":true},"execution_count":33,"outputs":[{"name":"stdout","text":"Model: \"model_3\"\n__________________________________________________________________________________________________\nLayer (type)                    Output Shape         Param #     Connected to                     \n==================================================================================================\ninput_7 (InputLayer)            [(None, 512, 1)]     0                                            \n__________________________________________________________________________________________________\nconv1d_149 (Conv1D)             (None, 512, 32)      512         input_7[0][0]                    \n__________________________________________________________________________________________________\nconv1d_150 (Conv1D)             (None, 512, 64)      30784       conv1d_149[0][0]                 \n__________________________________________________________________________________________________\nconv1d_151 (Conv1D)             (None, 512, 128)     123008      conv1d_150[0][0]                 \n__________________________________________________________________________________________________\nself_cal__conv1d_24 (self_cal_C (None, 512, 128)     262464      conv1d_151[0][0]                 \n__________________________________________________________________________________________________\nself_cal__conv1d_25 (self_cal_C (None, 512, 128)     344384      self_cal__conv1d_24[0][0]        \n__________________________________________________________________________________________________\nconv1d_152 (Conv1D)             (None, 512, 128)     49280       conv1d_151[0][0]                 \n__________________________________________________________________________________________________\nadd_9 (Add)                     (None, 512, 128)     0           self_cal__conv1d_25[0][0]        \n                                                                 conv1d_152[0][0]                 \n__________________________________________________________________________________________________\nmax_pooling1d_6 (MaxPooling1D)  (None, 256, 128)     0           add_9[0][0]                      \n__________________________________________________________________________________________________\nself_cal__conv1d_26 (self_cal_C (None, 256, 256)     385472      max_pooling1d_6[0][0]            \n__________________________________________________________________________________________________\nself_cal__conv1d_27 (self_cal_C (None, 256, 256)     1376896     self_cal__conv1d_26[0][0]        \n__________________________________________________________________________________________________\nconv1d_153 (Conv1D)             (None, 256, 256)     98560       max_pooling1d_6[0][0]            \n__________________________________________________________________________________________________\nadd_10 (Add)                    (None, 256, 256)     0           self_cal__conv1d_27[0][0]        \n                                                                 conv1d_153[0][0]                 \n__________________________________________________________________________________________________\nmax_pooling1d_7 (MaxPooling1D)  (None, 128, 256)     0           add_10[0][0]                     \n__________________________________________________________________________________________________\nself_cal__conv1d_28 (self_cal_C (None, 128, 512)     1540992     max_pooling1d_7[0][0]            \n__________________________________________________________________________________________________\nself_cal__conv1d_29 (self_cal_C (None, 128, 512)     5506304     self_cal__conv1d_28[0][0]        \n__________________________________________________________________________________________________\nconv1d_154 (Conv1D)             (None, 128, 512)     393728      max_pooling1d_7[0][0]            \n__________________________________________________________________________________________________\nadd_11 (Add)                    (None, 128, 512)     0           self_cal__conv1d_29[0][0]        \n                                                                 conv1d_154[0][0]                 \n__________________________________________________________________________________________________\ntransformer_3 (Transformer)     (None, 128, 512)     4205568     add_11[0][0]                     \n__________________________________________________________________________________________________\nglobal_average_pooling1d_3 (Glo (None, 512)          0           transformer_3[0][0]              \n__________________________________________________________________________________________________\ndense_54 (Dense)                (None, 256)          131328      global_average_pooling1d_3[0][0] \n__________________________________________________________________________________________________\ndropout_23 (Dropout)            (None, 256)          0           dense_54[0][0]                   \n__________________________________________________________________________________________________\ndense_55 (Dense)                (None, 256)          65792       dropout_23[0][0]                 \n__________________________________________________________________________________________________\ninput_8 (InputLayer)            [(None, 17)]         0                                            \n__________________________________________________________________________________________________\narc_face_3 (ArcFace)            (None, 17)           4352        dense_55[0][0]                   \n                                                                 input_8[0][0]                    \n==================================================================================================\nTotal params: 14,519,424\nTrainable params: 14,519,424\nNon-trainable params: 0\n__________________________________________________________________________________________________\n","output_type":"stream"}]},{"cell_type":"markdown","source":"# Model Testing","metadata":{}},{"cell_type":"markdown","source":"## KNN Testing Based ArcFace Loss","metadata":{}},{"cell_type":"code","source":"###### Testing Model - ArcFace Style\nwith tpu_strategy.scope():     \n\n    def normalisation_layer(x):   \n        return(tf.math.l2_normalize(x, axis=1, epsilon=1e-12))\n\n    #X_dev_flipped = tf.image.flip_up_down(X_dev)  \n    #x_train_flipped = tf.image.flip_up_down(X_train_final)\n\n    predictive_model = tf.keras.models.Model(inputs=model.input,outputs=model.layers[-3].output)\n    predictive_model.compile(tf.keras.optimizers.Adam(lr=1e-4),loss='categorical_crossentropy',metrics=['accuracy'])\n\nwith tpu_strategy.scope():\n    y_in = tf.keras.layers.Input((17,))\n\n    Input_Layer = tf.keras.layers.Input((512,1))\n    op_1 = predictive_model([Input_Layer,y_in])\n\n    ##Input_Layer_Flipped = tf.keras.layers.Input((224,224,3))\n    ##op_2 = predictive_model([Input_Layer_Flipped,y_in]) \n    ##final_op = tf.keras.layers.Concatenate(axis=1)(op_1)\n\n    final_norm_op = tf.keras.layers.Lambda(normalisation_layer)(op_1)\n\n    testing_model = tf.keras.models.Model(inputs=[Input_Layer,y_in],outputs=final_norm_op)\n    testing_model.compile(tf.keras.optimizers.Adam(lr=1e-4),loss='categorical_crossentropy',metrics=['accuracy'])\n\n##### Nearest Neighbor Classification\nfrom sklearn.neighbors import KNeighborsClassifier\nTest_Embeddings = testing_model.predict((X_dev,y_dev_ohot))\nTrain_Embeddings = testing_model.predict((X_train,y_train_ohot))\n\ncol_mean = np.nanmean(Test_Embeddings, axis=0)\ninds = np.where(np.isnan(Test_Embeddings))\n#print(inds)\nTest_Embeddings[inds] = np.take(col_mean, inds[1])\n\ncol_mean = np.nanmean(Train_Embeddings, axis=0)\ninds = np.where(np.isnan(Train_Embeddings))\n#print(inds)\nTrain_Embeddings[inds] = np.take(col_mean, inds[1])\n\n#Test_Embeddings = np.nan_to_num(Test_Embeddings)\n\n##### Refining Test Embeddings\n#for i in range(Train_Embeddings.shape[0]):\n#    for j in range(Train_Embeddings.shape[1]):\n#        if(math.isnan(Train_Embeddings[i,j])):\n#            Train_Embeddings[i,j] == 0\n#        if(Train_Embeddings[i,j]>1e4):\n#            Train_Embeddings[i,j] == 1e4\n\n##### Refining Train Embeddings    \n#for i in range(Test_Embeddings.shape[0]):\n#    for j in range(Test_Embeddings.shape[1]):\n#        if(math.isnan(Test_Embeddings[i,j])):\n#            Test_Embeddings[i,j] == 0\n#        if(Test_Embeddings[i,j]>1e4 or math.isinf(Test_Embeddings[i,j])):\n#            Test_Embeddings[i,j] == 1e4\n\n#del(X_train_final,X_dev,X_dev_flipped,x_train_flipped)\n#gc.collect()\n\nTest_Accuracy_With_Train = []\nTest_Accuracy_With_Test = []\n                                                                     \nfor k in range(1,11):\n    knn = KNeighborsClassifier(n_neighbors=k,metric='euclidean')\n    knn.fit(Train_Embeddings,y_train)\n    Test_Accuracy_With_Train.append(knn.score(Test_Embeddings,y_dev))\n    knn.fit(Test_Embeddings,y_dev)\n    Test_Accuracy_With_Test.append(knn.score(Test_Embeddings,y_dev))\n\nprint('--------------------------------')\nprint(np.max(Test_Accuracy_With_Train))\nprint(np.max(Test_Accuracy_With_Test))\nprint('--------------------------------')\nprint(np.mean(Test_Accuracy_With_Train))\nprint(np.mean(Test_Accuracy_With_Test))\nprint('--------------------------------')\nprint((Test_Accuracy_With_Train)[0])\nprint((Test_Accuracy_With_Test)[0])\nprint('--------------------------------')\n\nplt.plot(np.arange(1,11),np.array(Test_Accuracy_With_Train),label='Test_Accuracy_With_Train')\nplt.plot(np.arange(1,11),np.array(Test_Accuracy_With_Test),label='Test_Accuracy_With_Test')\nplt.title('Testing Accuracy vs Number of Neighbors')\nplt.xlabel('Number of Neighbors')\nplt.ylabel('Test Accuracy')\nplt.legend()       \nplt.show()  \n\nnp.savez_compressed('Test_Embeddngs_ECG1D.npz',Test_Embeddings)\nnp.savez_compressed('Train_Embeddngs_ECG1D.npz',Train_Embeddings)","metadata":{"execution":{"iopub.status.busy":"2021-07-28T08:42:04.583661Z","iopub.execute_input":"2021-07-28T08:42:04.584085Z","iopub.status.idle":"2021-07-28T08:42:11.217337Z","shell.execute_reply.started":"2021-07-28T08:42:04.584049Z","shell.execute_reply":"2021-07-28T08:42:11.216259Z"},"trusted":true},"execution_count":34,"outputs":[{"name":"stdout","text":"--------------------------------\n0.9459459459459459\n1.0\n--------------------------------\n0.8995495495495496\n0.9648648648648649\n--------------------------------\n0.9459459459459459\n1.0\n--------------------------------\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"<Figure size 432x288 with 1 Axes>","image/png":"iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAABKhUlEQVR4nO3dd3gU5fbA8e9JQhJ6B+mhC9J7UQERRVRE1CtVQMUKYuFarl71Wq7lhx0VUSkWEGwXuyCKgtTQpPcIoQaQEEpIO78/ZgJLTFkgm9kk5/M8ebLTz2w2c/adeYuoKsYYY0xGIV4HYIwxJjhZgjDGGJMpSxDGGGMyZQnCGGNMpixBGGOMyZQlCGOMMZmyBFEAicgREanjdRwmb4lIVxGJ9fD414rIDvfz1zIA+/9eRIb4ue4cEbk1i2VRIqIiEpa7ERY8liDymPvPk/6TJiLHfaYHnsX+/vaPoKolVHVr7kX9t2MOdf/BbgzUMQoCEXnSfZ/+4TMvzJ0X5WFogTIGGOF+/pZnXOie9yoRCfGZ94yITPJn56p6hapOzr1wTU4sQeQx95+nhKqWALYDV/vM+9jr+Pw0BDgI3JSXB82n3/gOAv8RkVCvAzkTZ/le1wLW5LBOVaDfWew7KOXTz6TfLEEECREJEZGHRWSLiBwQkekiUs5dFikiH7nzD4nIEhGpLCLPAhcBY90SyFh3fRWReu7rSSLypoh8KyIJIrJIROr6HPcyEdkgIvEi8paI/JpV0dxdvxbQBbgNuFxEzvNZFioi/3LPIUFElopIDXfZBSIyS0QOisheEfmXT3zP+OzjtNskIhIjIg+JyB/AUfcb+MM+x1grItdmiHG4iKzzWd5KRP4pIp9nWO91EXktk3N8SEQ+yzDvNRF53X09VES2uvvflkPJ7wcgCRiUxft5WgnQ3fc8n2kVkbtEZJN7vKdFpK6IzBeRw+7nJDzDPv8lIvvd926gz/wIERkjItvdv8E4ESnq+767574HmJhJrCEi8piI/Cki+0TkAxEp7e73CBAKrBSRLdm8Hy/iJMxML6wi0sE9t0MislJEumb2XrmftZfc89wmIiPk77eNaonI7+77NlNEKmQ43M0isktEdovI6Azv06vusl3u64is3icRqSAi37gxHxSRueJTSsrXVNV+PPoBYoBL3dejgIVAdSACeAeY6i67HfgaKIbzT9gaKOUumwPcmmG/CtRzX08CDgDtgDDgY+ATd1kF4DDQ1102CkjOuL8M+/43sNh9vQp4wGfZP915DQEBmgPlgZLAbuABINKdbu8T3zM+++gKxGZ4j1YANYCi7rwbcL6JhgA3AkeBKj7LdgJt3Rjq4XyzreKuV8ZdLwzYB7TO5BxrAceAku50qBt/B6C4+541dJdVAS7I4r16EvgI6A1sBYq4x1UgKrO/HzAUmJfhbzkDKAVcAJwAZgN1gNLAWmCIz3uXAryM8xnq4p5zeqyvAF8B5dy/wdfAcxm2fcHdtmgm53MzsNk9dgngC+DDzD53WbwfCtQHlqafM/AMMMl9XQ3ns9rL/dv2cKcrZnyvgDvcc68OlAV+cvcf5rPuFqABUNSdft5dFuWuO9X9ezYF4jj1v/gUzv9iJaAiMB94Oqv3CXgOGOf+fYvgfGkTr68vuXKN8jqAwvzD6QliHdDdZ1kVnIt1mPuPOR9olsk+Tv7T+MzLmCDe81nWC1jvvr4JWOCzTIAdGfeXYd+bgHvd148AK32WbQCuyWSb/sDyLPY3iZwTxM05vI8r0o8L/AiMymK974Hh7uurgLXZ7HMecJP7ugewxX1dHDgEXEcmF9EM+3gS+Mh9vQi4k7NLEJ19ppcCD/lMvwS86vPepQDFfZZPx0nqgpMs6vos6whs89k2CYjM5nxmA3f5TDdM/4xm/Nxlsb3iJOxewJ9AOKcniIfwSTg+f88hGd8r4Gfgdp/1LuXvCeIxn+V3AT+4r6Pcdc/3Wf4i8L77egvQy2fZ5UBMVu8TTkKZkd2559efglEMKhhqAV+6xdRDOAkjFagMfIjzj/KJW+R9UUSKnMG+9/i8Pobz7Q+cb+E70heo82nPshaMiHQGagOfuLOmAE1FpIU7XQPnnyujrOb7a4fvhIjcJCIrfN6rJjiloZyONZlTt3oG4byvWZmCk9gABrjTqOpRnFLLHcBucW7dne/HOTwGPIpTgjpTe31eH89kuoTP9F9ujOn+xPk7V8QpgS71ed9+cOeni1PVxGziqOruz3ffYTifUb+p6nc4n7PbMyyqBdyQHp8b44U4X5Yyi8X3c7Ejk3Wy+txntk36+5S+74znWdVnOuP79H84JauZ7q3HhzOJJV+yBBE8dgBXqGoZn59IVd2pqsmq+h9VbQx0wvn2m/6A+Fy6492NU0QHQETEdzoTQ3C+ia5w778u8pmffg51M9luB85ticwcxblwpTsvk3VOnqM4z0DeBUYA5VW1DLDajSu7GAD+BzQTkSY472F2lQI+BbqKSHXgWtwEAaCqP6pqD5wL13o3nmyp6iyci8hdGRb5c/5noqyIFPeZrgnsAvbjJJMLfD5fpdWpLHEyzBz2vQvnIu677xROT1j+ehT4F6ef+w6cEoTv/0BxVX0+k+1P++zifDE4U77bpL9PkPl57vKZPu19UtUEVX1AVevg3E68X0S6n0U8QccSRPAYBzzrXgARkYoico37upuINBWnJsxhnGJ9mrvdXrK++ObkW5wSQB/34d7dZHGBEpFI4B84D6db+PyMBAa4278HPC0i9cXRTETKA98AVUTkXvcBYEkRae/uegXQS0TKifPA+94cYi6O8w8a58Y1DKcEke49YLSItHZjqJf+nrrf+j7DudgvVtXtWR1EVeNwblNMxLkNs849XmURuca9CJ8AjnDqb5GTR4EHM8xbAfQVkWLiVCy4xc99Zec/IhIuIhfhJMJPVTUNJ5G9IiKV3HOpJiKXn8F+pwL3iUhtESkB/BeYpqopZxqgqs7BSexDfGZ/BFwtIpe7D6Ej3YfCmX1pmQ6Mcs+hDM7tqTP1b/d9vwAYBkxz508FHnP/BysAj7uxZUpErnI/ZwLE45T8/f1MBDVLEMHjNZwHiDNFJAHnIVn6RfQ8nAvbYZxbT79y6vbIa8D1IvKXuLVs/KWq+3Ee6r6I8zCwMRCNc+HLqA/ON9APVHVP+g8wAec2Q0+ch6PTgZlurO/j3KdPwLmPfzVOsX8T0M3d74fASpxnDTM59U+aVcxrce67L8BJjk2B332Wfwo8i5MEEnBKDeV8djHZ3Sa720vppuDc257iMy8EuB/nG+VBnAfBd/qxL1T1d2Bxhtmv4NzT3uvGdq5VnfcAf7nxfQzcoarr3WUP4ZRiForIYZwHuw3PYN8TcN6334BtQCLOF4Sz9Rg+fxtV3QFcg1OyiMMpUfyTzK9T7+J8Xv4AlgPf4ZRmUs/g+L/ivB+zgTGqOtOd/wzO/8EfOJUulrnzslIf5708gvO5fEtVfzmDOIKWuA9ZjMGtmhcLDCwoH/CMRKQmzm2h81T1sNfxmNwhIlcA41S1Vo4rG79ZCaKQc4vzZdx63v/CuZe/0OOwAsJNgPfjVPO15JCPiUhREeklTruYasATwJdex1XQFOhWgMYvHXFuoYTj1Cvvo6rHvQ0p97nPDPbi1Ejp6XE45twJ8B+cW5LHcZ6nPe5pRAWQ3WIyxhiTKbvFZIwxJlMF5hZThQoVNCoqyuswjDEmX1m6dOl+Va2Y2bICkyCioqKIjo72OgxjjMlXROTPrJbZLSZjjDGZsgRhjDEmU5YgjDHGZKrAPIMwJlglJycTGxtLYmJ2HaUaE1iRkZFUr16dIkX87wjaEoQxARYbG0vJkiWJiorC6c/NmLylqhw4cIDY2Fhq167t93YBu8UkIhPEGZZwdRbLRZwhHzeLyB8i0spn2RBxhljcJCJDMtvemPwiMTGR8uXLW3IwnhERypcvf8al2EA+g5hE9l0aXIHTC2J9nC6k3wYQZxzmJ3B6Mm0HPCEiZQMYpzEBZ8nBeO1sPoMBSxCq+htOd8hZuQan62hV1YVAGRGpgjO83yxVPaiqfwGzCGTfOWlpMPMxOHAuA54ZY0zB42UtpmqcPuRfrDsvq/l/IyK3iUi0iETHxcWdXRQHt8KyD2DchbDkPbC+qYwxBsjn1VxVdbyqtlHVNhUrZtpSPGcV6sGdC6BGe/j2AfjoOji8K+ftjMknDhw4QIsWLWjRogXnnXce1apVOzmdlJSU4/Zz5sxh/vz5fh2rRYsW9OvX71xD9syMGTPo06fPyennnnuOevXqnZz++uuv6d27N7t27eL6668HYMWKFXz33Xcn13nyyScZM2ZMjsc6179LdHQ099xzzxmc3ZnzshbTTk4fE7a6O28n0DXD/DkBjaR0NRj8pVOCmPlveKsDXPkyNL0+oIc1Ji+UL1+eFStWAM7Fq0SJEowePdrv7efMmUOJEiXo1KlTtuutW7eO1NRU5s6dy9GjRylevHi265+tlJQUwsICc+nq1KkTt99++8npBQsWUKpUKfbt20elSpWYP38+nTp1omrVqnz22WeAkyCio6Pp1avXGR3Ln79Ldufapk0b2rRpc0bHPFNeJoivgBEi8gnOA+l4Vd0tIj8C//V5MH0Z8EjAoxGBdsOh7iXw5e3w+S2w/hsnURQrl/P2xvjhP1+vYe2u3B2rqHHVUjxx9QVntM3SpUu5//77OXLkCBUqVGDSpElUqVKF119/nXHjxhEWFkbjxo15/vnnGTduHKGhoXz00Ue88cYbXHTRRZnuc+rUqQwePJh169YxY8YMBgwYAMCSJUsYNWoUR48eJSIigtmzZ1OsWDEeeughfvjhB0JCQhg+fDgjR4482adahQoViI6OZvTo0cyZM4cnn3ySLVu2sHXrVmrWrMlzzz3H4MGDOXr0KABjx449mcBeeOEFPvroI0JCQrjiiisYPnw4N9xwA8uWLQNg06ZN3HjjjSenfVWsWJFSpUqxefNm6tWrx86dO7nuuuuYP38+ffr0Yf78+TzzzDPExMRw1VVXsWzZMh5//HGOHz/OvHnzeOQR51K1du1aunbtyvbt27n33nvP6Jv+0KFDiYyMZPny5XTu3Jl+/foxatQoEhMTKVq0KBMnTqRhw4bMmTOHMWPG8M033/Dkk0+yfft2tm7delbHzErAEoSITMUpCVQQkVicmklFAFR1HM4Ysr1wxoQ9hjNoOKp6UESeBpa4u3pKVbN72J27yteFYT/A76/CnOfgz/nQeyw0uCzPQjAmkFSVkSNHMmPGDCpWrMi0adN49NFHmTBhAs8//zzbtm0jIiKCQ4cOUaZMGe644w6/Sh3Tpk1j1qxZrF+/njfeeIMBAwaQlJTEjTfeyLRp02jbti2HDx+maNGijB8/npiYGFasWEFYWBgHD+b8L7527VrmzZtH0aJFOXbsGLNmzSIyMpJNmzbRv39/oqOj+f7775kxYwaLFi2iWLFiHDx4kHLlylG6dGlWrFhBixYtmDhxIsOGDcvyOJ07d2b+/PmkpqZSv359OnTowI8//shVV13FypUradu2LXv27AEgPDycp556iujoaMaOHQs4pYH169fzyy+/kJCQQMOGDbnzzjvPqIFabGws8+fPJzQ0lMOHDzN37lzCwsL46aef+Ne//sXnn3/+t23O9ZiZCViCUNX+OSxX4O4slk3AGSDdG6FhcPFoqN8DvrgdptwArYfCZc9CRAnPwjL535l+0w+EEydOsHr1anr06AFAamoqVapUAaBZs2YMHDiQPn36nHYvPifp3/pr1qxJtWrVuPnmmzl48CA7d+6kSpUqtG3bFoBSpUoB8NNPP3HHHXecvH1SrlzOpfTevXtTtGhRwGmdPmLECFasWEFoaCgbN248ud9hw4ZRrFix0/Z76623MnHiRF5++WWmTZvG4sWLszxOp06dTiaIjh070q5dO5566imWL1/O+eefT2RkZI6xXnnllURERBAREUGlSpXYu3cv1atXz3G7dDfccAOhoaEAxMfHM2TIEDZt2oSIkJycHJBjZiZfP6QOuCrN4bY50OkeWDoZxnWG7QVyuGZTiKgqF1xwAStWrGDFihWsWrWKmTNnAvDtt99y9913s2zZMtq2bUtKSopf+5w6dSrr168nKiqKunXrcvjw4Uy/5eYkLCyMtLQ0gL816vJ9pvHKK69QuXJlVq5cSXR0dI4Pda+77jq+//57vvnmG1q3bk358uWzXDe9BDF//nw6duxIyZIlSUxMZM6cOTk+h0kXERFx8nVoaKjf72M633P997//Tbdu3Vi9ejVff/11lo3dzvWYmbEEkZMikXDZ0zDsO6cK7ISeMOsJSDnhdWTGnJWIiAji4uJYsGAB4HwbX7NmDWlpaezYsYNu3brxwgsvEB8fz5EjRyhZsiQJCQlZ7i8tLY3p06ezatUqYmJiiImJYcaMGUydOpWGDRuye/dulixx7hgnJCSQkpJCjx49eOedd05exNJvMUVFRbF06VKAbBNMfHw8VapUISQkhA8//JDU1FQAevTowcSJEzl27Nhp+42MjOTyyy/nzjvvzPb2EkCjRo3YtWsX8+bNo2XLloBTO2vcuHF07tz5b+vn9P6cq/j4eKpVc2r6T5o0KWDHyYwlCH/V6gR3/g6tbnKeT4zvBntWeR2VMWcsJCSEzz77jIceeojmzZvTokWLk7dUBg0aRNOmTWnZsiX33HMPZcqU4eqrr+bLL7+kRYsWzJ0792/7mzt3LtWqVaNq1aon51188cWsXbuWAwcOMG3aNEaOHEnz5s3p0aMHiYmJ3HrrrdSsWZNmzZrRvHlzpkyZAsATTzzBqFGjaNOmzclbLJm56667mDx5Ms2bN2f9+vUnv3H37NmT3r1706ZNG1q0aHFaddOBAwcSEhLCZZdl/zxRRGjfvj3ly5c/eQ+/Y8eObN26NdMSRLdu3Vi7di0tWrRg2rRp2e77bDz44IM88sgjtGzZMldKBWdCtIA0DGvTpo3m2YhyG3+EGSPg+F/Q7V/QeRSEZP1hNoXbunXraNSokddhFHpjxowhPj6ep59+2utQPJPZZ1FElqpqpvVlrTfXs9HgcrhrIXx7H8z+D2z8Afq87dSAMsYEnWuvvZYtW7bw888/ex1KvmIJ4mwVLw83TIZVn8F3DzhddVz2DLS52WlTYUwB9Oyzz/Lpp5+eNu+GG27g0Ucf9Sgi/3z55Zd/m3fttdeybdu20+a98MILXH755bl+/AMHDtC9e/e/zZ89e3a2D8y9ZreYckP8TphxN2z9BepdCr3fgFJVc97OFAp2i8kEizO9xWQPqXNDelcdvcZAzO/wVkenZGGMMfmYJYjckt5Vxx3zoHw9p6uOT4fBsbxrBG6MMbnJEkRuq1APbv4RLnkM1n3llCY2/eR1VMYYc8YsQQRCaBhc/E8Y/jMULQsfXwdf3wsnjngdmTHG+M0SRCCd7KpjJCydZF11GE/YeBD+y0/jQcCZ/W3OhiWIQCsS6VR/HfotaBpMvAJ+etK66jB5Jn3cgRUrVnDHHXdw3333nZwODw/PcXt/L0IZx4MIlEC2Ju7UqRMLF576Euc7HgSQ5XgQvgnCX+f6dwFLEAVHVGe4cz60HATzXoF3L4E9q72OyuS17x+GiVfm7s/3D59xGEuXLqVLly60bt2ayy+/nN27dwPw+uuv07hxY5o1a0a/fv2IiYlh3LhxvPLKK1l2tZEufTyIyy67jBkzZpycv2TJEjp16kTz5s1p164dCQkJpKamMnr0aJo0aUKzZs144403AKcvpv379wNOD7Fdu3YFnG/lgwcPpnPnzgwePJiYmBguuugiWrVqRatWrU67SL7wwgs0bdqU5s2b8/DDD7NlyxZatWp1cvmmTZtOm/blOx4EcNp4EOAkiM6dOxMTE0OTJk1ISkri8ccfZ9q0aad1tZE+HkSdOnV4/fXXPf/bnC1rKJeXIko6bSQaXglfjYTxXeGSR53eYq2rDpNHbDyI4B0PIjk5OSB/m7NlCcILDXs6XXV8c69zu2nDD3Dt21CujteRmUC74nmvI7DxIIJ4PIgNGzbk+t/mXNgtJq8ULw//+AD6vgv71sHbFzpjYqf493DKmLNl40EE73gQgfjbnIuAJggR6SkiG0Rks4j87UapiNQSkdki8oeIzBGR6j7LXhSRNSKyTkReFymAHRyJQLN/wF0LoEZb+PYBeKkBfHUPbP0V0lK9jtAUQDYeRPCOB9GwYcNc/ducq4AlCBEJBd4ErgAaA/1FpHGG1cYAH6hqM+Ap4Dl3205AZ6AZ0ARoC3QJVKyeK10NBn0JA6ZD/ctg9efwQW946Xz47kHYvgjcb1XGnCsbDyJ4x4MIDw/P1b/NuQpYZ30i0hF4UlUvd6cfAVDV53zWWQP0VNUdbgkhXlVLuduOBS4EBPgNGKyq67I6nqed9eW25OPOmBOrP4dNMyElEUrXgAuuhSbXOe0rCmCBqqCyzvqCg40HEVzjQVQDdvhMxwLtM6yzEugLvAZcC5QUkfKqukBEfgF24ySIsZklBxG5DbgNoGbNmrl/Bl4pUhQu6OP8JB6GDd87yWLhWzD/dShX10kUTa6DSud7Ha0xQc/Ggzg7XtdiGg2MFZGhOKWEnUCqiNQDGgHpzyRmichFqnpaGUpVxwPjwSlB5FnUeSmyFDS/0fk5dhDWfe0ki7lj4LcXodIF0KSv82O1oEyA2XgQZ8fGg8i4Yz9uMWVYvwSwXlWri8g/gUhVfdpd9jiQqKovZnW8AnWLyR8Je2HtDCdZ7HBbflZt5ZQqLrjWea5hgsK6des4//zzKYj1LEz+oaqsX78+aMaDWALUF5HaIhIO9AO+yhBYBRFJj+ERYIL7ejvQRUTCRKQIzgPqLJ8/FEolK0P72+CWH+He1dDjaacrj5mPwiuNYcIVsPhdOBLndaSFXmRkJAcOHKCgDM5l8h9V5cCBA3614fAV0BHlRKQX8CoQCkxQ1WdF5CkgWlW/EpHrcWouKc4tprtV9YRbA+ot4GJ32Q+qen92xyp0JYisHNgCq7+A1Z9B3HqQEKjdxSlZNLrK6V3W5Knk5GRiY2P/Vq/fmLwUGRlJ9erV/9aiO7sShA05WpDtXevcglr9GfwVAyFFnCFRm1wHDa+AiBJeR2iM8ZhXtZiM1yo3dn4ueQx2LXNLFl/Axu8hrCg0uNxJFvV7ODWnjDHGh5UgCpu0NOeh9urPYc3/4Nh+CC8J518JTa+HOl0hNOdOxYwxBYPdYjKZS02BmN+cZLHua0iMd55RXNAXOo+CsrW8jtAYE2CWIEzOUk7Alp+dZLH2K6dGVKub4OLRUKpqztsbY/Ilr6q5mvwkLMJ5cH3de3DPcmg1GJZNhtdawA//suqyxhRCliDM35WuBle9AiOXOs8lFr0NrzWHn/7jtOY2xhQKliBM1spGQZ+34O7FziBH8152EsWcF5w+oowxBZolCJOzCvXh+glwx+9Q+2KY8194rRnMexWSAjc4vTHGW5YgjP/OawL9PobhP0O11vDTE84zioXjILmAtRJOPAwrp8HUAfDpUOfBfUE7R2NyYLWYzNn7cwH8/Az8OQ9KVYOL/wktB+XfdhRJx5zxN3zH4ShV3fmd3l6k0VVO40JrL2IKCKvmagJHFbb96iSK2CXOc4suDztDqYZkPSJY0EhJOlW9d8N3kHQEilc6NThT9bZOld/M2os0vsZZp1bn/HGuxmTCEoQJPFXnW/fPT8OeVVChAXR9BBr3gZAgu5OZmgIxc90L/lfOBT+yzKkLftSFWV/wfduLrP8Oko9CicqnJxTr1tvkI5YgTN5JS4P1X8Mv/3V6k63cBLo96rSx8PLCmZYGOxa5DQH/B0fjILwEnO9zyygs/Mz2mXQMNrlDw26cCaknoHRNaOImi/OaWbIwQc8ShMl7aanOhfOX/8Jf25zBjC55DOpekncXTVXYtdztd+pLOLwTwiKhQc/c76Qw8bBzi2r1504JIy0Fytc7NTRsxYa5cxxjcpklCOOd1GRYORV+fRHid0DNTk6iiOocuGOe7Ob8cyc5ndbNeU+IKBm4Y4M7NOxXzvG3zQXUKUk16ev0c1WudmCPb8wZsARhvJdyApZ9AL/9HxzZ65Qkuj0G1Vvnzv5PDpT0OcStC56BkhL2+AwNu8iZV631qaFhrZ8r4zFLECZ4JB2D6Pdh3itw7AA0uAIueRTOa3rm+zq0w7l1tPpz2L3CmVezk/NNvfE1UKJSroZ+zg5t94l3JSBQq5OTLBpfA8UreB2hKYQ8SxAi0hN4DWfI0fdU9fkMy2vhjENdETgIDFLVWHdZTeA9oAbOsKO9VDUmq2NZgshnTiTAonHw+xtwIt6p7dTtXznfqz+yzxnHYvXnzrgWAFVbQpPr4YI+ULp6gAPPJfs3w5ovYNVnsH8DSKjzoLzJdc7YHEXLeB2hKSQ8SRDuuNIbgR5ALLAE6K+qa33W+RT4RlUni8glwDBVHewumwM8q6qzRKQEkKaqx7I6niWIfOr4X7DgTVj4NiQfg6b/gK4PQbk6p9Y5dtBpf7D6c6d6qqZBpcan7umXr+td/OdKFfb5PjOJgdBwqNfDOb+GV0B4ca+jNAWYVwmiI/Ckql7uTj8CoKrP+ayzBuipqjtERIB4VS0lIo2B8ap6ob/HswSRzx3dD7+/CovfdWoAtRgINTs49+83z4a0ZCdppNcKqtTI64hzn+rpQ8Mm7IIixdxaV32hbBA83A4Jc/rmsoaBBYZXCeJ6nIv/re70YKC9qo7wWWcKsEhVXxORvsDnQAXgIuBWIAmoDfwEPKyqqRmOcRtwG0DNmjVb//nnnwE5F5OHEvbA3JcgeqKTFEpVP9WuoEqLwtOuILOhYYNFifPchoF9rWFgARDMCaIqMBYnCfwGXAc0AS4F3gdaAtuBacB3qvp+VsezEkQBE7/Tqe1UpUXwtcTOa6kpsH2BczvOaycSnPYem2b5NAzs6zYMbGrJIh/KLkGEBfC4O3EeMKer7s47SVV3AX0B3OcM16nqIRGJBVao6lZ32f+ADjhJwxQGpas5PwZCw6D2RV5HcUrLgU73JOvdhoELxjq3B8vXd28B9rWGgQVEIBPEEqC+iNTGSQz9gAG+K4hIBeCgqqYBj+DUaErftoyIVFTVOOASwIoHxgSLyNLQor/zc/TAqYaBv74Avz4PlZu6JYu+TgeOJl8KdDXXXsCrONVcJ6jqsyLyFBCtql+5t6Gew6nG+htwt6qecLftAbwECLAUuE1Vk7I6lt1iMiYIJOw5VQ05drEzr1obt2FgH2sYGISsoZwxJu/99eephoF7/sBpGNj5VENGaxgYFCxBGGO8tX+TW333M9i/0RoGBhFLEMaY4KAKe9ecahh46E9rGOgxr2oxGWPM6UScsc3PawLdH4edy9y2Hl/Ahm99GgZe5/TAWyTS64gLNStBGGO8l5bmtPVIH9Dp2AGIKOUzoFMXGwM8QOwWkzEm/0hNccY5X/2F0wfXiXgoWs5nDPBO1tVHLrIEYYzJn1JOOH1xrf7cacGdfMzp6qPR1U4vvpUaOY3yCttzi7Q0iN8O+9Y5nT0WKQYd7jyrXdkzCGNM/hQWAef3cn6SjsJGdwzw5R/CknfdlcRpjFepsZMwKjVyXpevd+bjjAcbVafLmX1rTyWDfetg33pIPnpqvdoXn3WCyI4lCGNM/hBe/FTr7LRUOLjt7xfOjT9Aep+eIWFO9x/pCSM9eZSNCs5bVMcOQtx6n3Nyz8u3D67ilZxzaHXTqfOq2BAiSwUkpBwThIiEZuxF1RhjPBUSChXqOT+Ne5+an3LCaXPhmzR2LnVqSaULK+pcVE8mDfd3qap509lg0lE3Eaw7Pc6E3afWiSjtxNS4z+nJLY8bF/pTgtgkIp8DE30H+zHGmKATFnGqGq2vE0cgbsPpJY4tP8PKKafWSb8on1biaAzFy59dLClJcCBDstq31hkU6mS8brKq082bZJWDHB9Si0hJnI72hgEhOB3qfaKqhwMfnv/O5SG1qiJB8McwxuSxYwczXMDd14mHTq2TflvntBLH+RBR0lmelupc9DPe7jqw2Rn8CoL6dleu1WISkS7AFKAM8BnwtKpuzo0gz9XZJoiDR5O4dfISRl/ekE51rW8YYwo9VafTwYxJI269U4sqXemaULS0c0srJdGdmf8emJ9TLSZ3bOkrcUoQUTg9rH6MM+rbd0CDXIvUI0dPpHLLpGg+uKUdbaPKeR2OMcZLIlCqivNTr/up+WlpTtcgvqWExHio3eVUQihgVW79ucW0FfgFeF9V52dY9rqq3hPA+Px2LreY4hJOcOP4Bew7fIIPb2lHy5plczk6Y4wJTtmVIPwZy7GZqt6SMTkABEtyOFcVS0Yw5dYOlC8Rzk0TFrN6Z7zXIRljjOf8SRBvikiZ9AkRKSsiE7JZP186r3QkU4Z3oFRkEQa/v4j1e4LqGbwxxuQ5f0sQh9InVPUvoGXAIvJQtTJFmTK8PRFhoQx6bxGb9x3xOiRjjPGMPwkiRERO3pQXkXL42QJbRHqKyAYR2SwiD2eyvJaIzBaRP0RkjohUz7C8lIjEishYf46XG2qVL87Hw9sDwoB3FxKz/2iO2xhjTEHkT4J4CVggIk+LyDPAfODFnDZyaz+9CVwBNAb6i0jjDKuNAT5Q1WbAUzjjU/t6Gmes6jxVt2IJpgxvT0qaMuDdhew4eCznjYwxpoDJMUGo6gfAdcBeYA/QV1U/9GPf7YDNqrpVVZOAT4BrMqzTGPjZff2L73IRaQ1UBmb6caxc16ByST66pT1Hk1IZ8N5Cdscf9yIMY4zxjD8lCFR1DTAd+Ao4IiI1/disGrDDZzrWnedrJdDXfX0tUFJEyotICE7JZXR2BxCR20QkWkSi4+Li/AjpzDSuWooPbm7HoaPJDHh3EfsOJ+a8kTHGFBA5JggR6S0im4BtwK9ADPB9Lh1/NNBFRJYDXYCdQCpwF/CdqsZmt7GqjlfVNqrapmLFirkU0uma1yjDpJvbsvdwIgPfW8SBIycCchxjjAk2/pQgngY6ABtVtTbQHVjox3Y7gRo+09XdeSep6i5V7auqLYFH3XmHgI7ACBGJwXlOcZOIPO/HMQOida1yTBjalh1/HWPQ+4s5dCzJq1CMMSbP+JMgklX1AE5tphBV/QXItNVdBkuA+iJSW0TCcTr8+8p3BRGp4N5OAngEpyNAVHWgqtZU1SicUsYHqvq3WlB5qUOd8rx7Uxu2xB3hpgmLOZyY7GU4xhgTcP4kiEMiUgKnNtHHIvIakGPdT1VNAUYAPwLrgOmqukZEnhKR9A7cuwIbRGQjzgPpZ8/iHPLMRfUrMm5QK9btPszQCYs5ciLF65CMMSZg/OmLqThwHCeZDARKAx+7pYqgkZdjUv+weg93T1lG61plmTysHUXDg3B0KmOM8cNZ98XktmX4RlXTVDVFVSer6uvBlhzyWs8m5/HqjS2IjjnI8A+iSUy2AfeMMQVPtgnCHWo0TURK51E8+cbVzavyf9c35/ct+7nzo6UkpaR5HZIxxuQqf7rMOAKsEpFZ+Dx7KCg9uZ6L61pX50RKGv/6chUjpy5j7IBWFAn1q2lJ0EpITGbHweM0qlLSRtkzppDzJ0F84f6YTAxoX5Pk1DSe+GoN901bwWv9WhIakv8urMeSUpg8/0/e+W0Lh44l07JmGUZf1pDO9WyUPWMKqxwThKpOzotA8rMhnaJISknj2e/WER4WwpjrmxOST5JEYnIqUxZt5605m9l/JImuDStyYb0KvD9vGwPfW0SHOuUYfVlD2thIe8YUOv4MOboN+FtVJ1WtE5CI8qnhF9fhREoqY2ZuJDw0hP9e2zSok0RSShqfLt3BG7M3s+dwIh3rlOedwQ1oXctJBIM61GLq4u28+csWrh+3gC4NKvLAZQ1oVr2Mt4EbY/KMP7eYfKs/RQI3APZ1MhMjLqnPiZQ03vh5MxFhITzZ+4Kgu4+fkprG/1bs4rXZG9lx8Ditapbh5X80p1OGW0mRRUIZ1rk2N7atwQcL/mTcr1voPfZ3Lr+gMvf1aMD555Xy6AyMMXklx3YQmW7k1JttHYB4zlpetoPIjqry3PfrGf/bVoZfVJt/9WoUFEkiLU35dtVuXvlpI1vjjnJB1VKMvqwhXRtW9Cu+hMRkJsyL4b25WzmSlMLVzapy76X1qVOxRB5Eb4wJlOzaQfhzi6mVz2QITonCrwGDCiMR4ZErzicpJY13524jskgoD1zW0LN4VJWf1u3jpZkbWL8ngQaVSzBuUCsuv+C8M0pcJSOLMOrS+gzpVIvxv21l4u8xfPPHLq5rVZ17utenRrliATwLY4wX/LnQv+TzOgWnV9d/BCacgkFEeOLqxidvN4WHhjCye/08jUFV+W3Tfl6euYGVsfFElS/Ga/1acFWzqudUy6pMsXAe7Hk+N19Ym7fnbOHDhX/yvxU7ubFtDUZ0q895pSNz8SyMMV46q1tMwShYbjH5SktT/vnZH3y+LJZHrjif27vUzZPjLtp6gJdmbmRxzEGqlSnKqO716duqGmEBaKOxJz6Rsb9sYtqSHYgIgzvU4s6udalQIiLXj2WMyX3Z3WLypy+m/wIvut1w445P/YCqPpbbgZ6LYEwQAKlpyr3TVvD1yl08eXVjhnauHbBjLd/+Fy/P2sjcTfupVDKCkZfU4x9taxARFvi+onYcPMbrszfx+bJYIsJCGdY5itsurkOZYuEBP7Yx5uyda4JY7o7X4Dtvmaq2ymobLwRrggBITk1jxJRl/LhmL/+9tikD2vszIJ//1uyK55VZG/lp3T7KFQ/nrq51GdShFpFF8r4TwS1xR3jtp018/ccuSoSHcetFdbj5wihKRhbJ81iMMTk71wTxB9BWVU+400WBaFW9INcjPQfBnCDAaXdwx0dL+WXDPv7v+uZc37r6Oe9z874EXpm1iW9X7aZUZBi3d6nL0E5RFI/wvg7B+j2HeWXWRn5cs5cyxYpwR5e63NSxFsXCvY/NGHPKuSaIh4CrgYnurGHAV6r6Yq5GeY6CPUGA02p5+AfR/L55P6/2a0nv5lXPaj9/HjjKaz9t4n8rdlK0SCi3XFibWy6qQ+miwfct/Y/YQ7w8ayNzNsRRoUQEd3erS/92NT0p3fhDVYk7coKNe46wYW8Cm/cl0KpmWW5oUyPnjY3Jh84pQbg76Alc6k7OUtUfczG+XJEfEgTA8aRUhk5cTPSff/HmgJb0bFLF7213HjrO2J83MT06liKhwpCOUdzepS7ligf/ff7omIOMmbmBhVsPUqV0JCMvqc8Nbap72rlh/LFkNu5LYMOeBDbuPfX7r2OnRgssHh7K0aRUhnaK4t9XNc6X/WwZk51zLUHUBnaraqI7XRSorKoxfhy4J/AaEAq8p6rPZ1heC2eY0YrAQWCQqsaKSAvgbaAUkAo8q6rTsjtWfkkQAEdPpHDThMX8EXuIcYNa071R5WzX33c4kbfmbGHKou2A00HgXV3rUqlU/qtSOn/zfsbM3MCy7YeoWa4Y915an2taVAvohfd4Uiqb9zklgg17DrNh7xE27klgz+HEk+uUiAijQeUSNDyvJA0ql6Rh5ZI0OK8kZYuF89x363hv3ja6n1+J1/u3DIpbeMbklnNNENFAJ1VNcqfDgd9VtW0O24UCG4EeQCzOGNX9VXWtzzqf4gxINFlELgGGqepgEWkAqKpuEpGqwFKgUXpNqszkpwQBcDgxmUHvLWL97gTeG9KGixtU/Ns6B48m8c6vW5i8IIbkVOUfbaoz4pL6VCtT1IOIc4+qMmdDHGNmbmDNrsPUrVic+3o0oFeTKufUf1Vyahrb9h/9W4ngz4PHSP+Yh4eFUL9SiZMJIP131dKR2TYc/HBBDE98tYZGVUoxYWhbKufD5GxMZs41QaxQ1RYZ5q1U1eY5bNcReFJVL3enHwFQ1ed81lkD9FTVHeL8d8ar6t86+RGRlcD1qropq+PltwQBcOhYEv3fXcS2/UeYOLQdHeuWByD+eDLvz93K+/O2cSw5lWtbVOOe7vWJqlDc44hzl6ry45o9vDRzI5v2HaFRlVI80KMB3RtVyvZinZam7Pjr2KlE4JYItu4/QnKq83kODRFqVyjuJIDKJWl4XgkaVC5JzXLFzro9yC/r9zFiyjJKFS3C+0Pa0riq9Udl8r9zTRCzgDdU9St3+hrgHlXtnsN21+Nc/G91pwcD7VV1hM86U4BFqvqaiPQFPgcq+A5pKiLtgMnABaqaluEYtwG3AdSsWbP1n3/+me25BKMDR07Q/92FxP51nHGDWrNqZzzv/LqFw4kpXNm0CvdeWp/6lUt6HWZApaYp3/yxi1dmbSTmwDGa1yjD6MsacGG9Cuw9fIINexPYuCfB+b03gU17j3DcZ5jXGuWK+iQC53edisUD0v5j7a7D3DxpCQmJyYwd2IpuDSvl+jGMyUvnmiDqAh8DVQEBdgCDVXVLDtv5kyCqAmOB2sBvwHVAE59GeVWAOcAQVV2Y3fHyYwki3b6ERPq9s5Ct+50B+y5tVIn7ejTggqqFa6TXlNQ0vli2k9dmb2LnoeMUCw/lWNKpRFCpZMTfnhHUr1Qiz58J7IlP5JbJS1i3+zD/uaYJgzvUytPjG5ObzrkWk7uTEgCqekRE2qrqkhzWz/EWUyb7X6+q1d3pUjjJ4b+q+llO8eXnBAHOReed37bQu3lVWtYs63U4njqRksqn0bFs3JtA/UrOraEGlUtSNohqax09kcI9U5cze/0+br2wNo/0amQ1nEy+lFsJojHQH+iH86wg0x36rB+G85C6O7AT5yH1AFVd47NOBeCgqqaJyLNAqqo+7j4I/x74WlVf9Se+/J4gTP6TmqY8/c1aJs2P4bLGlXm1XwtrCGjynewSRLZP60QkSkQecVtTfwjcCfTIKTkAqGoKMAL4EVgHTFfVNSLylIj0dlfrCmwQkY1AZeBZd/4/gIuBoSKywv1pkdMxjclLoSHCk70v4ImrG/PTur30G7+QfQmJOW9oTD6RZQlCRBbgtEP4BPjErXK6TVUD19vcObAShPHST2v3MnLqcsoVD2fC0LY0PK9gVywwBcfZliD2AiVxvtmnV9IvGH2DG5PLLm1cmU/v6EhyahrXvz2f3zbGeR2SMecsywShqn2ApjiN1J4UkW1AWbfaqTEmgybVSvO/uztTrWxRhk1awtTF270OyZhzku0zCFWNV9WJqnoZ0B74N/CKiOzIk+iMyWeqlinKZ3d24qL6FXjki1U8//160tKs4G3yJ7+blKrqPlUdq6qdgQsDGJMx+VqJiDDeu6kNgzrUZNyvWxgxdRmJPg37jMkvzqrPAVXNf02WjclDYaEhPH1NEx67shHfr95Dv/EL2X/khNdhGXNGvOtr2ZgCTkS49aI6vD2wNev3HKbPm7+zaW+C12EZ47ccE4SIdPZnnjEmcz2bnMe02zqSmJxG37fnM3/zfq9DMsYv/pQg3vBznjEmC81rlOF/d3eiSulIbpqwmOnRVs/DBL8s+wVw+1LqBFQUkft9FpXCGQDIGHMGqpctxmd3duLuj5fx4Gd/sP3AMe7v0eCcxsAwJpCyK0GEAyVwkkhJn5/DwPWBD82YgqdUZBEmDG1L/3Y1GPvLZkZNW2E1nEzQyrIEoaq/Ar+KyKT0WksiEgKUUNXDeRWgMQVNkdAQ/nttU6LKF+e579ez69Bxxg9uTfkSEV6HZsxp/HkG8ZyIlBKR4sBqYK2I/DPAcRlToIkIt3epy1sDW7F6Zzx9357PlrgjXodlzGn8SRCN3RJDH5wuuGsDgwMZlDGFRa+mVZh6WweOJKbQ9635LNp6IOeNjMkj/iSIIiJSBCdBfKWqyVinfcbkmlY1y/K/uztToUQ4g95fxBfLYr0OyRjAvwTxDhADFAd+E5FaOA+qjTG5pEa5YnxxZ2faRpXj/ukreWXWRvwdzMuYQMkxQajq66paTVV7qeNPoFsexGZMoVK6WBEmDWvHDa2r89rsTdw/fSUnUqyGk/GOPy2pK4vI+yLyvTvdGBgS8MiMKYTCw0J48fpm/PPyhny5fCeD31/MoWNJXodlCil/bjFNwhk2tKo7vRG415+di0hPEdkgIptF5OFMltcSkdki8oeIzBGR6j7LhojIJvfHEpIpNESEu7vV443+LVmx4xB935pPzP6jXodlCqEsE4SIpLeRqKCq04E0ODnWdI7lXhEJBd4ErgAaA/3d0oevMcAHqtoMeAp4zt22HPAEzhgU7YAnRKTsGZyXMfne1c2rMuXW9vx1LIlr3/qdZdv/8jokU8hkV4JY7P4+KiLlcWsuiUgHIN6PfbcDNqvqVlVNwhnb+poM6zQGfnZf/+Kz/HJglqoeVNW/gFlATz+OaUyB0iaqHF/e1ZnSRYsw5P3F/BF7yOuQTCGSXYJI7yDmfuAroK6I/A58AIz0Y9/VAN8eyWLdeb5WAn3d19cCJd1k5M+2iMhtIhItItFxcTYGsCmYoioUZ+ptHShTvAiD31/M2l1WidDkjewSRHonfV2BL4EXcRrKvQtcmkvHHw10EZHlQBdgJ37cvkqnquNVtY2qtqlYsWIuhWRM8KlSuihTbu1A8fBQBr2/yMaVMHkiuwQRitNZX0mcNhBh7rxi7ryc7ARq+ExXd+edpKq7VLWvqrYEHnXnHfJnW2MKmxrlijFleAfCQoQB7y1iq3XNYQJMsmqMIyLLVLXVWe/Yeci9EeiOc3FfAgxQ1TU+61QADqpqmog8C6Sq6uPuQ+qlQPrxlwGtVfVgVsdr06aNRkdHn224xuQbm/cdod/4BYSFhDD99o7ULF/M65BMPiYiS1W1TWbL/HkGcVbc2k4jcKrIrgOmq+oaEXlKRHq7q3UFNojIRqAy8Ky77UHgaZyksgR4KrvkYExhUq9SCT66tT2JKan0f3chOw8d9zokU0BlV4Iol58uylaCMIXN6p3x9H93IeWKhzP99o5ULhXpdUgmHzqrEkR+Sg7GFEZNqpXmg5vbsT/hBAPeXUhcwgmvQzIFjD8tqY0xQaplzbJMHNaOXYcSGfTeIg4etW45TO6xBGFMPteudjneH9KGmANHGfz+IuKPJXsdkikgLEEYUwB0qleBdwa3ZtPeI9w0cTEJiZYkzLmzBGFMAdG1YSXeHNiKNTvjGTZxCUdPpHgdksnnLEEYU4D0aFyZ1/u3ZNn2v7h1cjSJyTaehDl7liCMKWB6Na3Cy/9owcJtB7jtw6WWJMxZswRhTAHUp2U1XujbjN82xjFiyjKSUtK8DsnkQ5YgjCmg/tG2Bk/3acJP6/Yx6pPlpKRakjBnxhKEMQXY4A61+PdVjfl+9R4e+HQlqWmZ95xgTGbCcl7FGJOf3XJhbZJS0njhh/WEh4bwwnXNCAk5p67WTCFhCcKYQuDOrnU5kZLKqz9tIjwshGf6NEHEkoTJniUIYwqJUd3rcyIljbfnbCE8LITHr2psScJkyxKEMYWEiPDg5Q05kZzGhN+3ER4WwsM9z7ckYbJkCcKYQkRE+PdVjUhKTeWdX7cSGRbKfT0aeB2WCVKWIIwpZESEp3o3ISkljddmO88k7u5Wz+uwTBCyBGFMIRQSIjzXtxlJKWn8348biAgL4daL6ngdlgkyAW0HISI9RWSDiGwWkYczWV5TRH4RkeUi8oeI9HLnFxGRySKySkTWicgjgYzTmMIoNEQYc0NzejU9j2e+XccHC2K8DskEmYCVIEQkFHgT6AHEAktE5CtVXeuz2mM4Y1W/LSKNge+AKOAGIEJVm4pIMWCtiExV1ZhAxWtMYRQWGsJr/VqSlLKMx2esISIshBvb1vQ6LBMkAlmCaAdsVtWtqpoEfAJck2EdBUq5r0sDu3zmFxeRMKAokAQcDmCsxhRaRUJDeHNgS7o0qMjDX6ziy+WxXodkgkQgE0Q1YIfPdKw7z9eTwCARicUpPYx0538GHAV2A9uBMZmNkS0it4lItIhEx8XF5XL4xhQeEWGhvDO4NR3rlOeB6Sv59o/dXodkgoDXfTH1ByapanWgF/ChiITglD5SgapAbeABEfnbEzRVHa+qbVS1TcWKFfMybmMKnMgiobw3pA2ta5Vl1CfLmblmj9chGY8FMkHsBGr4TFd35/m6BZgOoKoLgEigAjAA+EFVk1V1H/A70CaAsRpjgGLhYUwY2pYm1Upz95Rl/LJ+n9chGQ8FMkEsAeqLSG0RCQf6AV9lWGc70B1ARBrhJIg4d/4l7vziQAdgfQBjNca4SkYWYfLN7Wh4Xklu/2gpv2/e73VIxiMBSxCqmgKMAH4E1uHUVlojIk+JSG93tQeA4SKyEpgKDFVVxan9VEJE1uAkmomq+kegYjXGnK500SJ8eHN76lQozi2Tl7Bo6wGvQzIeEOd6nP+1adNGo6OjvQ7DmAJl/5ET3PjOAvbEJ/Lhre1pVbOs1yGZXCYiS1U101v4Xj+kNsYEsQolIpgyvAMVS0YwZMJiVsXGex2SyUNWgjDG5GjXoeP8450FHD6eTIsgKEWUiAhlSMco2tcp73Uo+V52JQjri8kYk6OqZYoydXgH/j1jNYeOJXsdDmt3xfPdqj1cVL8C9/doQMsgSFoFkZUgjDH5TmJyKh8t/JO35mzh4NEkLm1Uift7NKRx1VI5b2xOk10JwhKEMSbfOnoihUnzY3jn1y0cTkzhyqZVuK9HfepVKul1aPmGJQhjTIEWfzyZ9+du5f152zienEqfFtUYdWl9apUv7nVoQc8ShDGmUDh4NIl3ft3C5AUxpKQqN7SpzohL6lOtTFGvQwtaliCMMYXKvsOJvDVnC1MWbQdgQPua3NWtLpVKRnocWfCxBGGMKZR2HjrO2J83MT06liKhwpBOUdxxcV3KFg/3OrSgYQnCGFOoxew/yuuzN/Hlip0UDw/j5gtrc8uFtSldtIjXoXnOEoQxxgCb9ibw6k+b+HbVbkoXLcJtF9dhaKcoikcU3iZhliCMMcbHml3xvDJrIz+t20f54uHc2bUugzrUIrJIqNeh5TlLEMYYk4nl2//i5VkbmbtpP5VLRTDikvrc2KYG4WGFp5s6SxDGGJONhVsP8NLMDSyJ+YtqZYoy6tL69G1ZjbDQgp8orDdXY4zJRoc65Zl+e0c+uLkdFUqE8+Bnf9Djld+YsWInaWkF40v02bAEYYwxgIhwcYOK/O/uzowf3JqIsBBGfbKCK16byw+r91BQ7racCUsQxhjjQ0S47ILz+O6ei3ijf0uS09K446Ol9B77O79s2FeoEkVAE4SI9BSRDSKyWUQezmR5TRH5RUSWi8gfItLLZ1kzEVkgImtEZJWIWBNIY0yeCQkRrm5elZn3XsyYG5pz6HgSwyYu4fpxC5i/pXCM0x2wh9QiEgpsBHoAsThjS/dX1bU+64wHlqvq2yLSGPhOVaNEJAxYBgxW1ZUiUh44pKqpWR3PHlIbYwIpKSWNT5fu4I3Zm9lzOJFOdcvzwGUNaF2rnNehnROvHlK3Azar6lZVTQI+Aa7JsI4C6R24lwZ2ua8vA/5Q1ZUAqnogu+RgjDGBFh4WwsD2tZjzz648flVjNu5N4Lq3FzBs4mJW7yyYQ7EGMkFUA3b4TMe683w9CQwSkVjgO2CkO78BoCLyo4gsE5EHMzuAiNwmItEiEh0XF5e70RtjTCYii4Ry84W1+e3BbjzU83yWbT/EVW/M444Pl7JhT4LX4eUqrx9S9wcmqWp1oBfwoYiE4AyFeiEw0P19rYh0z7ixqo5X1Taq2qZixYp5GbcxppArFh7GnV3rMvehbtx7aX3mbd5Pz9d+Y9Qny9m2/6jX4eWKQCaInUANn+nq7jxftwDTAVR1ARAJVMApbfymqvtV9RhO6aJVAGM1xpizUiqyCPde2oC5D3bjji51mblmL5e+/CsPfraSHQePeR3eOQlkglgC1BeR2iISDvQDvsqwznagO4CINMJJEHHAj0BTESnmPrDuAqzFGGOCVNni4TzU83x+e7AbQzpG8b8Vu7jkpTn8+3+r2Xs40evwzkpAu9pwq62+CoQCE1T1WRF5CohW1a/cmkvvAiVwHlg/qKoz3W0HAY+4879T1UyfQ6SzWkzGmGCyO/44Y3/ezLQlOwgNEQZ3qMUdXetSoUSE16GdxvpiMsYYj+w4eIzXZm/ii2WxRBYJZVjnKG67qC6liwXHWBSWIIwxxmNb4o7w6k+b+HrlLkpGhnHbRXUYdmFtSng8FoUlCGOMCRLrdh/mlVkbmbl2L2WLFeHOrnUZ3CGKouHejEVhCcIYY4LMyh2HeHnWRn7dGEfFkhGM6FaPfu1qEBGWt4nCEoQxxgSpJTEHGfPjBhZtO0jV0pHc070+17WuTpE8GovCxoMwxpgg1TaqHJ/c1oGPbmlPpVKRPPzFKi59+Ve+XB5LqsdjUViCMMYYj4kIF9avwJd3dWLC0DYUDw/jvmkrufzV3/hu1W7PBi2yBGGMMUFCRLjk/Mp8M/JC3hrodB5x18fLuOqNecxetzfPx6KwBGGMMUEmJETo1bQKP957Ma/c2JyjSSncMjmaa9+az7xN+/MsUViCMMaYIBUaIlzbsjo/3d+F5/s2Zd/hRAa9v4h+4xeyJOZgwI9vtZiMMSafOJGSyieLdzD2l83EJZzg4gYVeaBHA5rXKHPW+7RqrsYYU4AcT0rlw4UxvD1nC38dS+bKplUYO6AlInLG+8ouQXjbxtsYY8wZKxoeym0X12VA+1pMnLeNxJTUs0oOObEEYYwx+VSJiDBGdq8fsP3bQ2pjjDGZsgRhjDEmU5YgjDHGZMoShDHGmEwFNEGISE8R2SAim0Xk4UyW1xSRX0RkuYj84Q5RmnH5EREZHcg4jTHG/F3AEoSIhAJvAlcAjYH+7hjUvh4DpqtqS6Af8FaG5S8D3wcqRmOMMVkLZAmiHbBZVbeqahLwCXBNhnUUKOW+Lg3sSl8gIn2AbcCaAMZojDEmC4FMENWAHT7Tse48X08Cg0QkFvgOGAkgIiWAh4D/ZHcAEblNRKJFJDouLi634jbGGIP3DeX6A5NU9SUR6Qh8KCJNcBLHK6p6JLvWgao6HhgPICJxIvJnHsQcSBWA/V4HEUTs/TidvR+n2HtxunN5P2pltSCQCWInUMNnuro7z9ctQE8AVV0gIpE4J9oeuF5EXgTKAGkikqiqY7M6mKpWzMXYPSEi0Vn1iVIY2ftxOns/TrH34nSBej8CmSCWAPVFpDZOYugHDMiwznagOzBJRBoBkUCcql6UvoKIPAkcyS45GGOMyX0BewahqinACOBHYB1ObaU1IvKUiPR2V3sAGC4iK4GpwFAtKN3LGmNMPhfQZxCq+h3Ow2ffeY/7vF4LdM5hH08GJLjgNN7rAIKMvR+ns/fjFHsvTheQ96PAjAdhjDEmd1lXG8YYYzJlCcIYY0ymLEEEARGp4fZJtVZE1ojIKK9j8pqIhLp9dH3jdSxeE5EyIvKZiKwXkXVum6FCS0Tuc/9PVovIVLd6fKEhIhNEZJ+IrPaZV05EZonIJvd32dw4liWI4JACPKCqjYEOwN2Z9FtV2IzCqf1m4DXgB1U9H2hOIX5fRKQacA/QRlWbAKE4VegLk0m47cd8PAzMVtX6wGx3+pxZgggCqrpbVZe5rxNwLgAZuyUpNESkOnAl8J7XsXhNREoDFwPvA6hqkqoe8jQo74UBRUUkDCiGTx9uhYGq/gYczDD7GmCy+3oy0Cc3jmUJIsiISBTQEljkcSheehV4EEjzOI5gUBuIAya6t9zeE5HiXgflFVXdCYzBaWS7G4hX1ZneRhUUKqvqbvf1HqBybuzUEkQQcTsp/By4V1UPex2PF0TkKmCfqi71OpYgEQa0At52u8U/Si7dPsiP3Hvr1+AkzqpAcREZ5G1UwcVtbJwr7RcsQQQJESmCkxw+VtUvvI7HQ52B3iISg9NF/CUi8pG3IXkqFohV1fQS5Wc4CaOwuhTYpqpxqpoMfAF08jimYLBXRKoAuL/35cZOLUEEAXG6rH0fWKeqL3sdj5dU9RFVra6qUTgPH39W1UL7DVFV9wA7RKShO6s7sNbDkLy2HeggIsXc/5vuFOKH9j6+Aoa4r4cAM3Jjp5YggkNnYDDOt+UV7k+vnDYyhcZI4GMR+QNoAfzX23C845akPgOWAatwrmGFqtsNEZkKLAAaikisiNwCPA/0EJFNOKWs53PlWNbVhjHGmMxYCcIYY0ymLEEYY4zJlCUIY4wxmbIEYYwxJlOWIIwxxmTKEoQJOiKiIvKSz/Rod2zy3Nj3JBG5Pjf2lcNxbnB7Xv0lw/wo9/xG+swbKyJDc9jfHSJyUw7rDBWRTMduF5EjZxC+MYAlCBOcTgB9RaSC14H4cjuH89ctwHBV7ZbJsn3AKBEJ93dnqjpOVT84g+PnmjM8b1OAWIIwwSgFp/HTfRkXZCwBpH8zFpGuIvKriMwQka0i8ryIDBSRxSKySkTq+uzmUhGJFpGNbt9P6eNP/J+ILBGRP0Tkdp/9zhWRr8ikBbOI9Hf3v1pEXnDnPQ5cCLwvIv+XyfnF4XTJPCTjAhGpKyI/iMhS97jnu/OfFJHR7uu2bowr3JhX++yiqrv9JhF5McO+X3HHUZgtIhXdeS1EZKG7vy/TxxEQkTki8qqIROMksxvcc1wpIr9lck6mALIEYYLVm8BAt7trfzUH7gAa4bRMb6Cq7XC6DR/ps14U0A6nS/Fx7oAzt+D0DNoWaAsMF5Ha7vqtgFGq2sD3YCJSFXgBuASnhXNbEemjqk8B0cBAVf1nFrG+AIwWkdAM88cDI1W1NTAaeCuTbScCt6tqCyA1w7IWwI1AU+BGEanhzi8ORKvqBcCvwBPu/A+Ah1S1GU7L5Cd89hWuqm1U9SXgceByVW0O9M7inEwBYwnCBCW3N9sPcAaH8dcSd2yNE8AWIL0b6FU4SSHddFVNU9VNwFbgfOAy4CYRWYHT1Xp5oL67/mJV3ZbJ8doCc9yO41KAj3HGbvDn/La6xxmQPs/tzbcT8KkbxztAFd/tRKQMUFJVF7izpmTY9WxVjVfVRJwSTy13fhowzX39EXChm3zLqOqv7vzJGeKf5vP6d2CSiAzHGaTHFAJ2b9EEs1dx+tyZ6DMvBfeLjYiEAL738U/4vE7zmU7j9M96xv5lFBCcb+4/+i4Qka44XWwHwn9x+hVKv0CHAIfcksHZ8n0PUsn6f9yfPnZOnreq3iEi7XFKXUtFpLWqHjj7ME1+YCUIE7RU9SAwHef2T7oYoLX7ujdQ5Cx2fYOIhLjPJeoAG4AfgTvdbtcRkQaS88A8i4EuIlLBvVXUn1MX+xyp6nqcb/lXu9OHgW0icoMbg4hI8wzbHAIS3Is1+D/cZgiQ/uxmADBPVeOBv0TkInf+4KziF5G6qrpIVR/HeYZSI7P1TMFiJQgT7F4CRvhMvwvMEJGVwA+c3bf77TgX91LAHaqaKCLv4dyGWiYignMR7JPdTlR1t4g8DPyCUwL5VlXPtJvlZ4HlPtMDgbdF5DGc5PcJsDLDNrcA74pIGs4FPd6P4xwF2rn73YfznAKcB+XjRKQYzu22YVls/38iUh/nPGdnEpMpgKw3V2PyGREpoarptbceBqqo6iiPwzIFkJUgjMl/rhSRR3D+f/8EhnobjimorARhjDEmU/aQ2hhjTKYsQRhjjMmUJQhjjDGZsgRhjDEmU5YgjDHGZOr/AaH4WuAWj33pAAAAAElFTkSuQmCC\n"},"metadata":{"needs_background":"light"}}]},{"cell_type":"markdown","source":"## t-SNE ","metadata":{}},{"cell_type":"code","source":"####### t-SNE Plot Generation\n###### Model Creation\n#with tpu_strategy.scope():          \n#    tsne_model = tf.keras.models.Model(inputs=model.input,outputs=model.layers[-4].output)\n#    tsne_model.compile(tf.keras.optimizers.Adam(lr=1e-4),loss='categorical_crossentropy',metrics=['accuracy'])\n#tsne_model.summary()\n\n###### Model Predicted\n#embeddings_final = tsne_model.predict((X_dev,y_exp))\n\n###### t-SNE plot plotting\n##### Reduction to Lower Dimensions\ntsne_X_dev = TSNE(n_components=2,perplexity=30,learning_rate=10,n_iter=2000,n_iter_without_progress=50).fit_transform(Test_Embeddings)\n\n##### Plotting\nj = 0 # Index for rotating legend\nplt.rcParams[\"figure.figsize\"] = [12,8]\nmStyles = [\".\",\",\",\"o\",\"v\",\"^\",\"<\",\">\",\"1\",\"2\",\"3\",\"4\",\"8\",\"s\",\"p\",\"P\",\"*\",\"h\",\"H\",\"+\",\"x\",\"X\",\"D\",\"d\",\"|\",\"_\",0,1,2,3,4,5,6,7,8,9,10,11,0,1,2,3,4,5,6,7,8,9,10]\nfor idx,color_index,marker_type in zip(list(np.arange(89)),sns.color_palette('muted',47),mStyles):\n    plt.scatter(tsne_X_dev[y_dev == idx, 0], tsne_X_dev[y_dev == idx, 1],marker=marker_type)\n#plt.legend([str(j) for j in range(89)])\nplt.savefig('tsne_plot_5000_iters.png')\nplt.savefig('tsne_plot_5000_iters.pdf')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-28T08:42:23.613897Z","iopub.execute_input":"2021-07-28T08:42:23.614304Z","iopub.status.idle":"2021-07-28T08:42:25.389715Z","shell.execute_reply.started":"2021-07-28T08:42:23.614269Z","shell.execute_reply":"2021-07-28T08:42:25.388777Z"},"trusted":true},"execution_count":35,"outputs":[{"output_type":"display_data","data":{"text/plain":"<Figure size 864x576 with 1 Axes>","image/png":"iVBORw0KGgoAAAANSUhEUgAAAsgAAAHSCAYAAADxDj0WAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAA9YElEQVR4nO3de3ycdZ3//fd3JoemOTShSZu0pZZaRFCkS9NyEIQCImsFgaKAiLigbIv+7OMnv5sb3X2svdVFwHW1euOugC7KDQILsqJ1ERBUoCdSNlqgVkpp6SHpuU2T5jjzvf9IJr0ymfPMNddcM6/n49FHmmsO1zePNu27n36+n6+x1goAAADAsIDXCwAAAAAKCQEZAAAAcCAgAwAAAA4EZAAAAMCBgAwAAAA4EJABAAAAhzKvF+DU2NhoZ82a5fUyAAAAUOTWr1+/z1rbFOuxggrIs2bNUltbm9fLAAAAQJEzxmyL9xgtFgAAAIADARkAAABwICADAAAADgRkAAAAwIGADAAAADgQkAEAAAAHAjIAAADgQEAGAAAAHAjIAAAAgAMBGQAAAHAgIAMAAAAOBGQAAADAgYAMAAAAOBCQAQAAAAcCMgAAAOBAQAYAlIT+oz36jy8vVf/RHq+XAqDAEZABAL6WavDd8uorOrBzu7b8T1ueVgbAr8q8XgAAANlwBt+TP3jeuMdXrrhbb61fq9DQkCTp6Xv+Vc/+6Pt697wztGjZbfleLgAfICADAHwp1eB79tWf1p5tb6tr726FQyEFgkHVNk7RB6++3qulAyhwtFgAAHzp7Ks/rdrGKQoEg5IUN/g2NE/TBz95ncKhkMorJygcCumDn7xO9c0tXiwbgA8QkAEAvpRO8N20+kWVVVTq7E98SmUVldq0+iUPVgzAL2ixAAD4ViT4nrX4Gq1+4hFtWv2S3nPmOeOeN//Sxbrg75aour5BJ5+7UEf27/NgtQD8goAMAPCd/qM9evgf/48uvHFJSsG3ec57Rn9eXd+g6vqGfC0VgA/RYgEAyIn29nZ1d3dLkrq7u9Xe3u7avSKTKw7t2a3Hvv5V9R/tUXV9g5rffWJa78NsZACxUEEGAGStq6tLv/rVr1ReXq7m5mZ1dnZqcHBQs2fPVl1dXc7uEz254rl7fyBrrX5xx9d07Tf/Je33SzYiDkBpMtZar9cwqrW11ba1McAdAPxo7969evDBB9XV1aW6ujpdf/31ampqyuk9Dnbu0n/d/Q0d3LVDzr+/TCCgsvLylGcbO4N2ZPRbsKyM2chACTHGrLfWtsZ6jBYLAEBONDU16fLLL5ckXX755TkPx9KxyRUyZvjHiGBZWVqzjSMj4kxg+K9BEwgwGxnAKAIyACBnzEhoNY7wmmubVr+o8soJOuWc80evhYaG0pptPDoibqRVI5zm6wEUN3qQAQA5U19fr/POO0/19fWu3WP+pYsVHgpp48t/GL1mw2H9esXdOmntqoQtEpHpF5OnH6/NbWtG2zSstSm9HkBpoIIMAMiZhoYGLVy4UA0N7o1Ra57zHn3o+htVO7lRwfJySVKwvFy1k5tGWyTiTaeIbMqb9t5TRl5fMfL6CtU2NtFiAUASARkA4EMNzdN03qdvlA2HR0/RC4eGVDUyMcM5nUIa3pT3/c8s1tM//K4k6cWH/kM9Bw8qNDig8soJsuGQzrvu72ixACCJgAwA8Cnn8dGBYJm6D+zXE9/62pgg/PQ9/6rvf2axBvp6Vds4RYFgUJKGp1aUl6m8cgLHTwMYJyc9yMaYn0j6mKQ91tr3j1xbLunzkvaOPO2r1trf5OJ+AADMv3SxwqGQVv3nQwqHQpKkzs1/lbVWRsObBAPBoGobp2jhDTdrz9a3tPL731Z55QSFhgZ11uJr9b7zL+L4aQDj5KqC/ICkS2Jc/661du7ID8IxACBnmue8Rx/69I2qbZyiYNlwZThYVqaahuMko9HWi8h0CmfFuayiUrvffmv0yOlMTuEDULxyEpCttX+UdCAX7wUAQKpGx7WFQqOBuHZyY8zWifmXLtaN3/uRWi+9Ujd+70eaf9lij1cPoFC5Pebti8aYz0hqk3Srtfagy/cDAJSYSGX4rMXXaPUTj6i8coJu/N6PxrVONM95z+hrqusbRqvHABAtZ0dNG2NmSfq1owd5qqR9kqykb0hqsdbeGON1N0u6WZJmzpw5b9u2bTlZDwDA/9rb2zVnzhzV1NSou7tbmzdv1ty5c8c8p3PzX1Xb2KTq+gb1HDqoI/v30S4BIKlER027VkG21u52LOA+Sb+O87x7Jd0rSa2trblJ6wAA3+vq6tKvfvUrlZeXq7m5WZ2dnRocHNTs2bNVNzLOTaIyDCD3XBvzZoxxDpO8QtJrbt0LAFC42tvb1d3dLUnq7u5We3t7Sq+rq6vTkiVLVFFRoa1bt6qiokJLliwZE44BwA25GvP2c0nnS2o0xuyQ9DVJ5xtj5mq4xWKrpL/Pxb0AAP6RahU4nqamJl1++eX62c9+pssvv1xNTU15WDWAUpeTgGytvTbG5R/n4r0BAP4VqQI/+OCD2rp1q+rq6nTTTTelVQU2xoz5CABu4yQ9AICrIlVgSRlVgevr63Xeeeepvr4+94sDgBjcHvMGAEBWVeCGhgYtXLgw10sCgLioIAMAXFcsVeD12w7qnhc2a/02xvoDxYwKMgDAdcVQBV6/7aCuu3+NBobCqigL6KHPnal572KkHFCMqCADAJCCNVv2a2AorLCVBofCWrNlv9dLAuASAjIAFKFMZw+XynqcUm2bOHP2ZFWUBRQ0UnlZQGfOnpynFQLIN1osAKDIZDt7uNjXIw2H4jVb9qthYoW+/uvXU2qbmPeuBj30uTO1Zst+nTl7Mu0VQBEz1hbO6c6tra22ra3N62UAgO/t3btXDz74oLq6ulRXV6frr7/e00M2Cmk9zl7igDEKha2spKCRvnzxSfrCwjl5WUd4IKSu599Rz5oO1ZzVotqFMxWoCObl3gAkY8x6a21rrMdosQCAIpTt7OFiXo+zlzhsrYIBk/e2if4th9V55zr1vLxLti+k7pd2qfPOderfcjgv9weQGAEZAHwg3R7e9vZ29fX1SZL6+voKoue3UE7Ec/YSV5QF9PWPv19fvvikvE6l6FnXqfDRIdnBsCTJDoYVPjqknnWdebk/gMToQQaAAufs4a2pqVF3d7cGBwdH+3nnzp0b8/llZWWqr6/XL3/5Sw0NDXna8ysVzizkdHuJI/3K9B0DpYMeZADwgb179+pnP/uZjhw5IkmaNm2aDhw4oMHBQS1btmxc8M1lz297e7vmzJkzGs43b948LpQXK7dmHx94ZJOOtu8Zd33i3Ck67pqTsn5/AMkl6kGmggwAPtDU1KQrrrhCP/vZzzRx4kTt2rVLdXV1uummm8aE40iYbWpq0iWXXKLHHnssq57fQpxAkU+xZh/nIiBXL2hW318PyA6GZQfDMuUBmfKAqhc052DVALJFQAYAn4j07p5zzjl65plnxgXf6DC7c+dOSVJvb2/G96yrq9OSJUv04IMPauvWrTFDeTGL9CsPDoVzuomvcvYkNd++QEdeeEfdqztUc9Y01S48nikWQIFgkx4A+ESkh7e2tlbS+M1ukTBbUVGhrVu3qrKyUvPnz9e0adOyum8hTaDIt0i/shub+AIVQU36yAmavvxsTfrILMIxUEAIyADgEw0NDVq4cKFmzJgRd7ObM8xeeeWVWrRokRoaGrI+ya5QJlB4Yd67GvSFhXPYoAeUEFosAMBnIkE5nugwm4s+4kKZQAEA+UAFGQCKTHSYjW69qKio0JIlS9LqI46E8oYGqqgAih8BGQCKTKwwW8p9xACQLgIyAJSIUu4jBoB0EJABoETQRwwAqWGTHgCUiGSb+1AYONoa8B4BGQCAAuHW0dYA0kOLBQAABSLW0dYA8o+ADABAgYgcbR00yunR1gDSQ4sFAAAFInK0NT3IgLeoIAMAYsr2eGpkhqOt/a374AE9d/89+tHSG7xeCrJABRkAME4ujqfOJyY/wGvdBw9ozRM/12u//51kwwoNDXm9JGSBgAwAGCdyPPWDDz6orVu3qq6uTjfddFPBhmMmP8ArBOPiRIsFACAmvxxPzeQHeOmRf7pNf3ruaYUGBwjHRYSADACIyw/HUzP5AU656AFO5z2u/ca3ddqH/1ZlFRUKlvEf88WCX0kAQFx+OJ462eQH+pNLQy5aHTJ5j+r6Bl100y06a/G1Wv3Ez/X675+TDdNq4XcEZABAXH45nnreuxpihl/6k3OnUP+h4VUwjhYdlN9qW5v2e6BwEJABAEUrVn9yIYU7vyjkf2g88k+36fDe3ZK1nr5HRCQoX3TTLVm/F7xDDzIAwFfWbzuoe17YrPXbDiZ9Lv3JuVHIGyFz0QOcznsw57g0UEEGAPhGupVMTqbLjcg/NAaHwgX3D41c9ABbayVrVTmxWnMWnBXzPRjnVloIyAAA38ikZSJefzJS54d/aMTrAY4E27fWr9Pf/9tPx7wmVuiNfg+CcWkiIAMAfKOQK5nFzi//0IgE5VMvvEQP3LpU+3dsV7CsLK1qsLOP+P7/9bmc9SfDPwjIAADfcLuSWaiTGpC63Vu36L9/8G3t37F99Fp0AE5nU9613/g249tKEAEZAOArblUyC3lSA5KLVIX/9Ox/J31uOqGXOceliSkWAACosCc1ILnIkc+piITez/3gx3r/BRenNL0i+jU1x9HeU8wIyAAAiJFwfucc1RYIxg670SPaMgm9kddEb/hDcTG2gJrOW1tbbVtbm9fLAACUKHqQ/a374AG9+PADeuPF30s2POax8spKhcN2dFPerY/+2pM1onAYY9Zba1tjPUYPMgAAI/wyqQGxJdp8Fw6F6BtGymixAAAARSHRiXiEY6SDgAwAQIkKD4R06Om3tXP5Kh3+7dsKD4S8XlJWonuKg+XlkjGSxBHSSAstFgAAlKD+LYe1//97Q3YwLDsYVvdLu9SztlOTP32KKmdP8np5WYkezfbmutU6MeoI6chYuDdfWaMT55/JSXkYg016AAAkER4Iqev5d9SzpkM1Z7WoduFMBSqCXi8rKwce2aSj7XvGXZ84d4qOu+ak0c+L6WvvOXRQf3z4Ab255iWFQmGFQ0OSteNO2mMDX2lgkx4AABlKtdJaTEEyopiqzJGK8aZVL0o2rLAjEFM1RjQCMgAAcYQHQjr4X5sVPnosQEXCYs+6ztGQWExB0qlnXWfSr90vEk24iK4gA2zSAwDAIbJxbcc/vaxdX1+job1Hk74mEiTt4PDsXTsYVvjokHrWdbq93IxVL2hWYGKZTPlwFDDlAQUmlql6QXPK7+GnTX6JJlykepoeSge/EwAAGOGsBGswnPwFPlY5e5Kab1+gIy+8o+7VHao5a5pqFx6fcltItlXzeC0pbrWqRG/cc27Yc15/q21t1veC/7FJDwCAEfE2ro1hpEBV2ZggmOqGN7+JhODwQEgaGskLZUaNn3mfjr66J+OvOTpcm/KATHlAtRfO1JHfvTPuuhutKj2HDo4GYo6NLk2JNunRYgEAQBrKpkxU8+0LxgS2XLQrFKLK2ZPUcM1JkszoNWOMDjzyF4W6BzJ+33gtKd0v7cxbq0qkokw4Riy0WAAASlb0f+fbcPz/VY1UMxs+Pmfcf/ln265QyHpf3SsNHWs3iVR3h/b3ergqwF0EZABASYrVQ6uAkakMSmE7XMUsG66cmoBRzQenJwy9gYqgJn3kBE36yAn5/DI8Uza5SrY/NK4dwu9Vc0AiIAMASlSsEWaSVPWBRpVNnlB0leBcC9ZUaHKGVfPqBc3q++uBceG65pzpMXuQCd3INwIyAAAOJhAoqUpwMvHCbPWC5oyr5olaUqrnNxdlqwr8hYAMAADicqu/Ol64LrVWFRSmnARkY8xPJH1M0h5r7ftHrh0n6VFJsyRtlfRJa+3BXNwPAIBsJaqMFqOVW1Zqxasr1NnTqebqZi07fZkWzV6U0msJrSg1uRrz9oCkS6Ku3S7pd9baEyX9buRzAAAKQqQyWnPONJkJQdWcM33c+LZisXLLSi1ftVwdPR2ysuro6dDyVcu1cstKr5cGFKScHRRijJkl6deOCvImSedbazuMMS2Sfm+tTTg5nINCAADIvYsfv1gdPR3jrrdUt+iZq57xYEWA97w6KGSqtTby3dgpaaqL9wIAAHF09sQ+aCPedaDU5eUkPTtcpo5ZqjbG3GyMaTPGtO3duzcfywEAoKQ0V8fuq453HSh1bgbk3SOtFRr5GPNwe2vtvdbaVmtta1NTk4vLAQCgNC07fZkmBCeMuTYhOEHLTl/m0YqAwuZmQH5K0g0jP79B0i9dvBcAAIhj0exFWn72crVUt8jIqKW6RcvPXp7yFAug1ORkk54x5ueSzpfUKGm3pK9J+i9Jj0maKWmbhse8HUj0PmzSAwAAQD4k2qSXkznI1tpr4zx0YS7eHwAAAMiXvGzSAwAAAPyCgAwAAAA4EJABAAAABwIyAAAA4EBABgAAABwIyACAorK7q8/rJQDwOQIyAKBoPL5+h869+wVt3nPE66UA8DECMgCgaJx/UpOqyoP6zI/XadXmfVr91n5tP3DU62UB8BkCMgCgaDTWVOqUljrtOtynT92/Vtfet0aPr9/h9bIA+ExOTtIDAKAQ7Ovu1xsdXZpeX6Vvf+IDMjKa0VDl9bIA+AwBGQBQNH6/aa96B0N6YulZmjOl1uvlAPApAjIAoGhcNW+Gzj2xUVPrJni9FAA+Rg8yAKCoEI4BZIuADAAAADgQkAEAAAAHAjIAAADgQEAGAAAAHAjIAAAAgAMBGQAAAHAgIAMAAAAOBGQAAADAgYAMAAAAOBCQAQAAAAcCMgAAAOBAQAYAAAAcCMgAAACAAwEZAAAAcCAgAwAAAA4EZAAAAMCBgAwAAAA4EJABAAAABwIyAAAA4EBABgAAABwIyAAAAIADARkAAABwICADAAAADgRkAAAAwIGADAAAADgQkAEAAAAHAjIAAADgQEAGAAAAHAjIAAAAgAMBGQAAAHAgIAMAkAJrrQ4+8qj+etbZOvjIo7LWer0kAC4hIANAESPU5Ubo8GFtX7JEu++6S6GDB7X7rru0fckShQ4f9nppAFxgCukPy9bWVtvW1ub1MgDA16y1OvToY9r7ve8p2NSoge07pL4+maoqTVwwX9PvvlvBSZO8XqavbP3UderdsEEaHBy9NhiU3mox+rclM7Ts9GVaNHuRhysEkC5jzHprbWusx6ggA0ARGVPpPHRIA29ulvr6JEm2t1c9q1Zr+9JbPF6l/1ScMEsKhcZcC4SlnZOljp4OLV+1XCu3rPRmcQByjoAMAEVk+9Jb1LNqtWxvb+wnhEKqmD07v4sqApMuvUyBqqox1/rLpRffZyRJfaE+rXh1hRdLA+ACAjIAFJFYlU6nQFWV6j72MfqS0zSxdZ4kyVRUaCAoDQSHr2883ow+p7On04ulAXBBmdcLAADkzqRLL9ORp3+rcE/P2AfKymQCAVlrdeAnP9bRV9pke3u1+667dOSF5+lLTsKUlen4e3+kvjc26p72e9Q10KW3pxrZwLGA3Fzd7OEKAeQSARkAioiz0ilJNhyWCQTUdOutMoGADv3nf6pn9ZrRzWbOvuRZDz/k2br9YOK8eZo4b55O/2CDlq9arr5Q3+hjE4ITtOz0ZR6uDkAuEZABoIg4K50RE045WRPnDQfnvr9sVP/mzWNfRF9yWiLTKla8ukKdPZ1qrm5migVQZAjIAFBkIpXOWGK1YASqqjTp0kvztbyisGj2IgIxUMQIyABQQqJbMKKvAwAIyABQUuK1YJhg0MNVAUBhISADQIlJ1IIBAGAOMgAUBWsts40BIEcIyADgc2OOlz54ULvvukvblyxR6PDhtN+LoA0ABGQA8L3o46Wds43TkcugDQB+RkAGAJ+Lebx0BrONcxW0AcDvXA/IxpitxpgNxph2Y0yb2/cDgELkZuvCpEsvU6Cqasy1TGYb5ypoS7RqAPC3fFWQF1pr51prW/N0PwAoGG63LjhnG0d+OK+nKldBm1YNAH7HmDcAcIG1VocefUx7V6xQoKZGg52d0uDg8GOO1oVZDz+U9b1yNds4V4eIbF96i3o3bHDt6wUAt+UjIFtJzxhjrKQfWWvvdT5ojLlZ0s2SNHPmzDwsBwAy4wy9TcuWqf7qT8oYM+55ocOHtfO223R03Suyvb0KdXXlrHUhnlzMNs5V0K44YZZ629vHXszx1wsAbjJu94UZY6Zba3caY6ZIelbS/7LW/jHWc1tbW21bG23KALyRKABHh15TVaWJC+Zr+t13Kzhp0pj32fqp68ZUUGMJVFdrxg9/qOozFrj6NXmhZ81a7fjCFxTu6Rm9VsxfLwB/Msasj9f+63oPsrV258jHPZKelMSfjgAKTrK+2XQmPMTc7CZJgUBWPcJ+kaueaADwiqstFsaYakkBa+2RkZ9fLOnrbt4TADKRrG82nbaBSZdepiNP/3ZMBdVMmKD6q65SxbveJSmz1gW/yFWrBgB4xe0e5KmSnhz5L8oySQ9ba592+Z4AkLZkAThW6I034SHWZjcTDGrqV24vmZCYi55oAPCKqwHZWrtF0mlu3gMA4tl1x1pVnTxZdRfOVLCuIuFzkwXgdCY8eF1BTXUzIQAgNsa8ASha4a4B9bR1qmf9blXPm5owKCcLwOmGXq8qqNGbCXffdZeOvPB8zM2EAIDYXJ9ikQ6mWADIpR23v3jsk6CRjEkYlI+uXz8uAPutTSDmBI3yclWdeioziAHAIdEUCyrIAEpDyEqy6lnXob43D6rltvnjnlIMfbPMIAaA7OXrqGkA8FbQSGUBVZ/RoilLi3drRO0FF0rR/cbGqPaChd4sCAB8iAoygOIWaa1oHWmtqE28Wc/v9t1/f8xT+/bdd79qL7jAm0UBgM8QkAEUpV13rJXKA6o6tVH1f3tC0QfjiMrZJ6ivvV1y7i8xRpXvfrdnawIAv6HFAkBRCncNSGGr3j/vU9dz7yjUNeD1kvJi0qWXKTBx4phrgYkTY85rBgDERgUZQPGKbMxLcdRbMUhnXjMAIDYCMoDil8IEi2Lh9SElAFAMCMgAil/URr10+PFUumIYVwcAXiIgAyheWU6w4FQ6AChNBGQARSlQV6GqUyZnNdpt+9JbxpxKZ3t71bNqtbYvvYVT6QCgiBGQARSlaV89I+v34FQ6AChNjHkDgDgmXXqZAlVVY64FqqoYmQYARY4KMgDEwcg0AChNBGQAvrXrjrWqOnmya7ONGZkGAKWJgAzAt8JdA2MOATn6+j5NfF9jTgMzI9MAoPQQkAH4m+O0PIWsetZ2qKetU9WtzUV/ah4AwB0EZADFIWTH/LyUjpcGAOQWUywAFIdg1Ol2ISsNhdWzrkN7/v1P3qwJAOBLBGQA/hY0UllA1fObY18/o0VTlp7mzdoAAL5EiwUA34o+La9nTUfWx0sDAEBABuBb0afl5eJ4aQAACMgAikYujpcGAIAeZAC+suuOtTr45GaFuga8XgoAoEhRQQbgK9GHgzDCDQCQawRkAP7jOBwkUVB2+yhqAEBxosUCgH8lmXUcqTZ33P0KbRkAgJRRQQbgX1Ej3WKKqjYbI1Wd2qhJl5xAVRkAEBMBGUBKCqpdIZNZxyNB2Uo6+uoeHf3zPnqYAQAxEZABpKRQNsdlPOt4JFRrKDz8+VC4IL4eAEDhISADSF2Km+PclPas46hqc8c/rz32WOTrWdehvjcPquW2+TldKwDAnwjIANJXAMEylZaPpNXmVHqYAQAlh4AMIH0FECxTafmIW23OpIcZAFAyCMgAUpdCsMzrZr4MWj4y7mEGAJQMAjKAlKQaLD3ZzJdGy0faPcwAgJJDQAaQkrSCZb438xVAywcAoHhwkh4A9yQ56S5rQSOVBVQ9v1kt//d8NVw+Z7S6veuOtRmdnpfp6wCglPUc7teD/7hKPYf7s3pOoSAgA3BPJMCe0aIpS0/L6VsH6ipiBuOITI+ZTvd1BGoAkNpWvq2u/X1q+83WrJ5TKGixAJB7eZgSkVLLRxqtHpHNhem+rlAOUEFhunfZHzTYH1J5ZVA3rzjP6+UAOdFzuF+/+PZ6Xfl/zVP1pEr1HO7XxtWdkpU2rupQ60dnqXpS5bjXJHtOIaGCDCCnklV2s5FxtTaFVo9I0E33dWOel0HFGsVtsD805iPgR9GtEdGV4LaVb8uGrSTJhu2YCnHktauffCvucwoRARlATk376hk5D8YRmbZNpNzqEbJjPzeSykzqLSJu91zDN+5d9gfds+R5mYCRJJmA0T1Lnte9y/7g8cqA9DkDcXQleO+OI9q4ulPhkT8/wyGrjas6xobpfX3667rxz9m740jB9iTTYgHAX9KZkJFtq4eVAjUVarh8TmrPZ5pGyYpupYhUjJ0VM4lKMvwnOhAP9ofG/L5+7sevj34eEakQt3501vBrJdmwxj3nuR+/Phq8z7v2pLx8PakiIAPwpySzj7M+ECQoVc9vSS3ocjJfyYtupSivDGqwPyQTMLJhO/qxvDLo5TKBtDnbJ8KhsP66rnM07IZDVgc6jo57TThk9faf9knWjgvPzudEXluIPckEZAD+lKRam/2BIEYazuAJcTJfaYtUjiMirRTllUF94d8v0D1Lnpc0XC37wr9fEPf1bOJDIYpUjyOtEdFVYEkKBI1OOWfauArwcO/x6tHXSlKwPKDrv3mWqidV6g8P/0VvvNyhcMiOVpwLqYpMDzIAf0kw+zinQjalXmI3e65R+KJbJqJbKSIV43iVYzbxoZA5q8fxjFaLU3htJAhHB+/ovuVCQAUZgG/krVpLLzFSEKkOxxIJxPGqwpHKsbMFI1J5ppIMN0SPZkvF23/eP6YCHFFdX6kTPjBZr724S+//0PSYld9Yr03UelFoVWQCMgDfyL5tIgl6iZEjyUIum/iQb85JFKmG0M/e+cGY1yPtE4lmGn/2zg+OaaOItGK0fnSWfvqVl8e1a0TCMwEZAAoIvcRIVXTfcbREG/Fi9SyziQ9uy/UhHbHmHsfqQY7VRjHYPyRrpfefN12yNmEV2ksEZABQHqrTKBqJwnGsjXjxXhuZdCHF38QH5EIqgTZV8YJvdOiO14O8ae1uyUpvvLxLZmQzdCFOsWCTHgAAaYhUeSOHgKQi1sEh0WE53ms4XATZyPWGuESb75zi9SBHJgOFh6xCoXDc13uNCjIAAGmI9Bc7N+mlWjmO7jmWjoXle5f9YUzvMhMukAuJAm0mVeR4wfet/9mrd17fP7oJMLp/OdK3HBp0NB/bY68vtCoyARkAgAxEWiRS6RuOPjjEKXqDHhMukEuJpklkEpDjbdz7w8N/0Wsv7oobvJONjGOKBQAARSCdsHrzivNS3qDHhAvkUrxAm0upbAKMNzIugikWAACUoFQ36HFMNfwmlU2A+QjquURABgAgC8mOi47VMpFog150jzMTLlDIUp1q4TcEZAAAspBsM12iDXqJgm86Pc6AV3K9CbBQEJABAMhAqpvpkrVMxKtAsyEPfpDrTYCFgoAMAEAGUt1Ml6xlgnFu8DO/9RanyvWAbIy5RNIKSUFJ91tr73T7ngAAuC3dzXTRLROMcwMKl6sB2RgTlHSPpA9L2iHpFWPMU9baN9y8LwAAbkt3M1106GWcG1C43D5qeoGkzdbaLdbaAUmPSPq4y/cEPLNyy0pd/PjF+sBPP6CLH79YK7es9HpJAFwWqQinu5ku+sjqyEc25QHeM9bGH9qc9Zsbc5WkS6y1nxv5/HpJZ1hrv+h4zs2SbpakmTNnztu2bZtr6wHctHLLSi1ftVx9ob4x1+sr63X7gtu1aPYij1YGoJClc2Q1gNwxxqy31rbGesztCnJS1tp7rbWt1trWpqYmr5cDZGzFqyvGhWNJOtR/SMtXLaeaDCCmTCvQANzj9ia9nZKOd3w+Y+QaUHQ6ezrjPtYX6tOKV1dQRQYwDhvygMLjdgX5FUknGmNOMMZUSLpG0lMu3xPwRHN1c8LHEwVoAABQOFwNyNbaIUlflPRbSRslPWatfd3NewK5kMlmu2WnL9OE4IS4jycL0AAAoDC4PgfZWvsbSb9x+z5ArnxzzTf16KZHRz/v6OnQ8lXLJSlhi0TksW+t/ZYODxwe89iE4AQtO31Z7hcLAAByzvNNekAhiQ7HEZEe4mQWzV6kl659SXeee6daqltkZNRS3aLlZy+n/xgAAJ/gqGlgxMotK2OG44h0eogXzV5EIAYAwKeoIAMjklWI6SEGAKA0EJCBEckqxPQQAwBQGgjIKHmRiRVW8U+VvPqkq2mZAACgRNCDjJIW73hop6tPulr/eOY/5nFVAADASwRklLR4x0NLUkt1i5advozKMQAAJYYWC5Q0TrcDAADRCMgoaYkmU0QOCEnlFD0AAFA8CMgoWSu3rNTRwaMJn5PqASEAAKB40IOMkpTK5rwI2jAAACgtVJBRkhJtzovGASEAAJQWAjJKUqpV4QnBCRwQAgBAiSEgoyTFqwobGU2qmCQjo5bqFi0/ezlj3gAAKDEEZJSkZacv04TghHHXraz6Q/361rnf0jNXPUM4BgCgBBGQUZIWzV6k5WcvV8CM/xZgcgUAAKWNgIyStWj2IllrYz7G5AoAAEoXY95Q0pqrm9XR0xHzOlAMznzoTPUM9Yy7Xl1WrTXXrfFgRQBQ+Kggo6TF6kVmcgWKSaxwnOg6AIAKMkpcZBPeildXqLOnU83VzVp2+jI256FkJas4U5EGUAoIyCh5i2YvIhADI5JVnKlIAygFtFgAAAAADgRkAAAAwIEWCwAoYtVl1XF7huP1EwNAqSMgA0ARS7Rx7tSfnprHlQCAf9BiAQBIqrqseszHeI8DQDGgggwAGGfDDRtiXmeUG4BSQAUZAAAAcKCCDADIOQ4UAeBnBGQA8KlsQ2iiCRfZ4kARAH5GQAYAn8o2hFLJBYDY6EEGAAAAHAjIAAAAgAMBGQAAAHAgIAMAco4DRQD4GZv0AMCn3JxCkS02AALwMwIyAPgUIRQA3EGLBQAAAOBABRkAwMl3AOBAQAaAIpFNyOXkOwA4hhYLACgShFwAyA0qyABQwuJVnQGglFFBBoASRjgGgPEIyAAAAIADLRYAgLjSOXSESRgAigUBGQCKRK5P1ttww4a0ns8mQQDFgoAMAEWCKi0A5AYBGQCKVCotD7muOgNAMSAgA0CRSqXlIVHVmZ5iAKWKgAwAJSxRCKanGECpYswbAJSwXIbgeG0ZtGsA8BsqyACAnKDtAkCxoIIMAAAAOBCQAXiut3eHtmxZod7eHV4vpajQ8gAAmaHFAoDn+vp26O2t31dDwxmqqprh9XKKRrYtD4yAA1CqCMgAPGdteMxHuC/e9IoIRrn5xK526akvSUN9UtkE6bLvS9Pmer0qwPdosQDgqe6eN/XGxtskSW9svE3dPW96tpZSavVIFI433LCBcOwHu9qlBxZJnX+S9m0a/vjAouHrALJCQAbgmf7+3XrllcsUCh1VQ8NZCoWO6pVXLlN//25P1hNp9ejrK/6AjCLw1Jekge6x1wa6h68DyAotFgA8U1k5Ve896ZuaPPk8VVQ0amBgn/bv/6MqK6em9Pre3h3q6HhCLS2Lc9K7TKsHfGWoL73rAFLmWgXZGLPcGLPTGNM+8uOjbt2rVO3u4g9B+F9Ly2JVVDRKkioqGtXScmXKr82m4hvdTlFIrR5ASsompHcdQMrcbrH4rrV27siP37h8r5Ly+PodOvfuF7R5zxGvlwJ4JpuKrzNcF1qrB5CSy74vVdSMvVZRM3wdQFZosfCp809qUlV5UJ/58Tr9yydOkzFGMxqqdPxxE71eGpAX0RXfuXP/QzXVJ455TqIWDGe4zrbVw49yPcIt3lQMpmG4aNpc6bMrmWIBuMBYa915Y2OWS/qspC5JbZJutdYejPG8myXdLEkzZ86ct23bNlfWU4yuvXeNVm/ZP/r5sgtP1P/+8Hs8XBGQH/39u7Vq9fkKBKpUW3uKjhx5Q+Fwr84+6/djQu3Bg2v06v9cp9P/5iE1NJw5er275021t/+d+vs7VFnZEjNcIz2n/vTUuI9tuGFDHlcCAKkxxqy31rbGeiyrCrIx5jlJzTEe+gdJ/ybpG5LsyMfvSLox+onW2nsl3StJra2t7qT1IrSvu19vdHRpen2Vvv2JD8houIIMlIJUK76xWjAi7RSBQJUaGs7SkSNv6JVXLhsXrgEApSurgGytvSiV5xlj7pP062zuhbF+v2mvegdDemLpWZozpdbr5QB519KyePTnsTb3xWvBiA7Xh7s2aOvWexQOD6Z031xPzkD++b4dhMNBANe5OcWixfHpFZJec+tehcytSRNXzZuhF29bSDgGYki26c45OSMc6tG+fc+OmYSR6MAQZiX7X7xDUhIdnlIwOBwEyAs3N+ndbYyZq+EWi62S/t7FexWkx9fv0Fef3KDffOmchEF2d1efptalP5Ynk9cApSCdTXeR9ou+vt3asmWFWloWj4bghoYzEm7uA/Iu3uEgP75Yanov1WQgR1yrIFtrr7fWnmqt/YC19jJrbYdb98pWrqu8kfdzTppYtXmfVr+1X9sPHB3zXMa1Ae5IZb6ysw3jzc3fHK0MxwvBzEqOL970i0ynYiCOeIeAhPqpJgM5VPJHTec6oD6+fofOuet5bd5zRI01lTqlpU67DvfpU/ev1bX3rdHj68f+t2wqIRpA7o1vw+iXJHUdeT1mCGZWcmJrrlujDTdsGPfDFz29fpLsEBCOmgZyouTnIOd6nnB336AGQ1bX3rtGX//4+/XnnYc0ta5S3716bsxJE5EQvXrLfn3q/rWSGNcGuCV6g12kDWNg8KDa229Qf3+PNm++Q8FgTcwJF6U2KxkF6LLvD1eJo9ssnDhqGshayQfkXAfUj502Td9YuVF7uwe09KFXJUkXn9yss9/dGPP5jGsDciOV6RLRvcUtLYvHjX07fLhd4fCA3nfKd2RMcEwITjY5A/6Q60NS8sp5OMjevwy3VkTjqGkgayUfkPMRUOsnlsd9LN64tkw37gGlKtHGuohYvcXJKsOE4OLj+7aPaXOlJX88NtHCWU3mqGkgJ0o+IOd6nvCv/rRLobDVlNpKfe+a2G0VTlfNm6FzT2wcE4Z/8tLbuvPpvySdfgHgmGTTJRIdTU1lGDkVb05xrucXc9Q04BrXjprORGtrq21ra8v7fdOp1iZ77uPrd+irv9ig3yzLLNw+vn6HvvKLP2tCWVC1E8py0hcNFLtkR0enejQ1kLV4Vd2PfVf69f8ef/2zKwm0gEcSHTVNQE5DvLnG0aE5m/aIfd39uvA7f1DvQEgDoWOVMDbuAbGlGn47Op4Y10ZBpRg59+8fGh63Fq2sShrqHX+9+bThdgkAeZcoIJd8i0U6Yk282LDzsP7lmU1jQnM2vcONNZU6cUqN2rYdHL22+PTpumoeR9oCsaQ6XYI2CuRFvAkSNpTe8wF4ioCchlgTLz5/7gk5GxO3u6tPwYDR67u6JEnfvuoDmtEwkfYKIAnCLwpGvAkSJpje851y3bsMICkCchriTbx4bWdX1mPiIu0byy48UUPhsB65+QydOTv2aDgAQIE6+4vSLz4//vq5t0ovfy/1iRORUNx3SDq8XXJuPn1gEb3LgMsIyGmINfEinTFxiXqTI+0b9/1xi6yVdhzs1fYDR6kcA4CfrPp/Y1/f+KvUJ07E2ujnFDktj95lwDUE5DQ4R7JFwm6qY+LibfCLcLZvSNL/+c8/68q/ma5PtB5PiwUAFKro9oe+Q7GfN9R3bH5xMk99KfFJeZH3A+CagNcL8JupdRP0+PodOvfuF7R5zxFdNW+GXrxt4bipFtGcG/xWbd6n1W/t1/YDR0cfd1aiT2mpkyT94n926tr71ujx9Tvc/8IAAOmJVHo7/yTt2zT88dD22M9N9XS7Xe3DJ+Qlw2l5gKsIyBmIDrtb9vaMhl1neHaKVIh3He7Tp+5fOy74RirR37tmrnYe6tX0+io9/Pkz9PPPn8kECwAoRDErvWHJRP3Vmux0u13tw+PhvvcB6b6FsY+PTuf9AGSNFosMxJpmEdmYF2sU3IyGKlVVBBP2KkfaN158c19OT/YDALgkXpvDpOOlCfWpTZ1I1m88KiDVj7wvUywA1xGQM5BoY1688Hz8cROTBt+pdRNiHj0NAChA8docJtTH7zWO7lkePJo8HAcrpZueIRQDeURAzkCijXnxwvPxx01MOfgSjgHABy77fuxjpRONbhtXLTbJ79P0XsIxkGcE5AwkqvImCs8EXwAoItPmpj66TYrTs2wT34N+Y8ATBOQMxQu7tEgAQAlJdXSblGA0m9HYoEy/MeA1ArILCMcAgHHi9SxPniOVT+QoaaCAEJABAMiHeD3Li+8nEAMFhjnIAADkQ6Rnufk0qfGk4Y+fXUk4BgoQFWQAAPIlnZ5lAJ6hggwAAAA4EJABAAAABwIyAAAA4EBABgAAABwIyAAAAIADARkAAABwICADAFAgOvsHvV4CABGQAQAoCJt6+nTGmjf0aMcBr5cClDwOCgEAwCPv9PZre9+ArKQvbXxHEwMBXTi5zutlASWPgAwAgEce7Tyg72zdPfr5OfU1aqzgr2bAa3wXAgDgkaubj9PZ9TWykpZtfEevdfdq78CgmirKvV4aUNLoQQYAwGXxNt/NrKrUBxtqdU5DrR4+7d3qDYf1woEjGb0XgNwhIAMA4KJUN9+dVD1Ba888RZ9sPi7r9wKQHVosAADIsUw33zVXjm+tYCMfkH8EZAAAciyTzXed/YMxAzIb+YD8o8UCAIAcu7r5OD0x9916fO67Nb2yfHTzXTyJWiec79VcUZb0vQBkj4AMACg5bm90S2Xz3Tu9/Xr54BG9dPCIrv3TW3FbJyLv1VRRrgODIXWHQkk38gHIDv9HAwAoKZt6+nRx2ybd/Z7jdXVL/A1xuRLZfBfdPpFK60R0/3F1MKAn/maOTqmpcn3dQCkjIAMAip7XG91i9RanMgM5VogmHAPuIyADAIpedNCcXzfR841uM6sqNbOqUpL08Gnv1sVtm/TCgSNjxrw5Q/QX3+AgESBf6EEGABQ950a3KeVleqXrqO7fsdfrZY2KNwPZ2X+8f3CQ/mMgT6ggAwCKQrwxadGsGf5oXF5PuqLXHt0WUhMM6om5c3RKLS0WgNsIyAAA30u28S66xWLBpGrdNKMpn0tMW8z+Y8IxkBcEZACAL6Wz8S56Q9xfe/qy7uVNtWKdqVQ28QFwBz3IAABferTzgBa3v6Wr2t/Srv5Bva+mKu7Gu1TmEqcj0cEeTtHzltOZv5zrNQNIHQEZAOBL6Z5WFxFvQ1wyqR7sEREdolMN1blcM4DM0GIBAPClVMakxZNJa0QmB3tUGqPasoBeOngk6/nL2bRzuN0OAhQbAnKhumO6NNA9/npFjfTVnflfDwAUsHin1eVSJgd7HF9Zrhtf2zr6eaxQ7bZ8nxwIFAMCcqGKFY4TXQeAEud2hTTdgz2WbXxHh4ZC+sn7Z6muLJjXjXZenxwI+B0B2Y+WTzr281gVZarPAOCqeBXrWCG6OxTWR5vq024DyUYq7SAA4uO7xSu5CrGx3iNR9TlZuAYApCRZxTo6ROejDSSCEXFAdphi4ZVCaKGgXQMAXBUdhvO1UY4RcUB2qCDnW7zKMQAgZUxlSF0+K9dAsaCCnG+phuOKGnfXAQA+lc084VJFOAbSQwW5UEX3Bjt7h3Ppjun0IQMoeExlAJBPBGS/qKiJv6kv1efGQrsHAB9gKgOAfMrqTxdjzCckLZd0sqQF1to2x2NfkXSTpJCkL1lrf5vNvYrCHdOTPydea0U6Vd5Ux74BgE8wlQFAPmXbg/yapCsl/dF50RhziqRrJL1P0iWSfmiMCWZ5L/9LFlKXH3an3YEWCgA+x1QGAPmUVQXZWrtRkowx0Q99XNIj1tp+SW8bYzZLWiBpdTb3K2psygOAlDCVAYDb3Grgmi5pjePzHSPXxjHG3CzpZkmaOXOmS8vxgVSrvJySBwCEYwCuShqQjTHPSWqO8dA/WGt/me0CrLX3SrpXklpbW22271f0Ujklj7AMAACQsaQB2Vp7UQbvu1PS8Y7PZ4xcQz6wIQ8AACBjbrVYPCXpYWPMv0qaJulESetculdhS2WCBP3HAAAABSPbMW9XSPqBpCZJK40x7dbaj1hrXzfGPCbpDUlDkr5grQ1lv1wfShSOlx+OfT1RnzEAAABcle0UiyclPRnnsX+W9M/ZvH/JStRnDCBtb553voZ27x53vWzqVJ34h9/nf0EAgILGMUR+k84peQAkSTULF+rQE09Ig4PHLpaXq+aCC/K6DoI6APgDAdlvnNMpEvU3RyZaJEPbBkpA4y1LdfjJJ+Uck2MCATXdsjSv6yiUoA4ASIyA7LVsjoGON8otUTiO1/cMFLHyKVM06YorjoXT8nJNuvJKlTU15XUdbgZ1qtMAkDvZHjWNZOJVaCPXaZcA8qLxlqUygeE/8jINpW+ed742vvfkcT/ePO/8lF4fCeoqHznkIodBvWbhwmPvO3pDqtMAkAkqyG7jwA6gIIxWkR99NONQmosWCWcVOZdtHoXSRgIAxYCAXAyyadMASkjjLUvV8/LLGYfGXITQeEE92xaJQmkjAYBiQItFMSAcAykpnzJFc559JuPQWD5lihQY+8em7e/Xm+d+KOU2C2k4aJfPmKEjzz472qYRKxxnUp3Oto0EAEBALi1MrACyVnvxxeMvphlkI0G99qKLxvcNO2RanZYxVI8BIAsEZK+5HVqXHz72g35oIGtTbv2yFAyOvTg4qEM//3laG/aksRXfcTJskYhUp6keA0Dm6EH2WqzQylHTQMEqnzJFdR/7mLp++csYD6ZfSY7uG1YoJIXDGbdIRKrTAIDMEZALEZVeoKBNufXLOvLf/y07MDDmeiahNnqqRe1HP6qup56iRQIAPESLRTFINmsZQE6VT5miSVdeOfyJMSMXM2uJiO4bnnLrl2mRAACPGWtt8mflSWtrq21ra/N6GQCQ1OCePdp6zbUa2rdPGhiQqazUnOeezajqO7hnj7Zd92nNevghqsYAkCfGmPXW2tZYj9FiAQAZKJ8yRSc+/zt1LP9/sjp8JPJezr5hjo0GAG8RkAEgC9kePhJLohP7CM8A4D4CMgBkyBlW3zz3Q6PXsw2rCU/sszbr464BAIkRkAEgQ4kqvalIVA2Od2x0Lo67BgAkxhQLAMhQzIM+HIeGOH/EOkCkZuHC8SfpjQTseMdGj069iLwuw+kZAID4CMgAkKFYYbVizpy4oTfizfPO18b3nqxDjzwytvqsY2E40bHR8cIzACA3aLEAgBTEa4cINjbKBAKjB31M//bd2nrNtQlbIGK2ZkhSIDAmDMfbADh6Al+W0zMAALFRQQaAFMRrh6j98IfHVHonnHxy0haImK0Zkkx5+ZgwHBn/FisAN96ylANFAMAlBGQASEGsUBupDEeH1WQtEONaM0ZO40unGpwoPAMAskNABoAUJNocFx1WE/UPR4wJ3OXlKps2jWowABQIAjIApCidzXHJWiCcIbp+8WKd+PzvqAYDQIFgkx4ApCidzXFbP/FJDe3ePeYAEWnsISJunMIHAMgeFWQASEOqm+MSzTge/ZQ+YgAoSARkAEhDqqE20aY+AEBhIyADgAs48Q4A/MtYa5M/K09aW1ttW1ub18sAgJwY3LNHb334Ytn+fpnKSgVqaxXat2/c85x9yQCA/DDGrLfWtsZ6jAoyALgketxb7UUXJe1LBgB4j4AMAC5ybuqjLxkA/IGADAAucm7qoy8ZAPyBgAwAeZTOYSMAAG8QkAEgj1I5hhoA4C0CMgDkWaqHjQAAvMFR0wCQZ5G+ZABAYaKCDAAAADgQkAEAAAAHAjIAAADgQEAGAAAAHAjIAAAAgAMBGQAAAHAgIAMAAAAOBGQAAADAgYAMAAAAOBCQAQAAAAcCMgAAAOBAQAYAAAAcCMgAAACAAwEZAAAAcCAgAwAAAA7GWuv1GkYZY/ZK2ub1OkpIo6R9Xi8CnuL3QGnj17+08euPUv898C5rbVOsBwoqICO/jDFt1tpWr9cB7/B7oLTx61/a+PUHvwfio8UCAAAAcCAgAwAAAA4E5NJ2r9cLgOf4PVDa+PUvbfz6g98DcdCDDAAAADhQQQYAAAAcCMglyBjzCWPM68aYsDGmNeqxrxhjNhtjNhljPuLVGpEfxpjlxpidxpj2kR8f9XpNyA9jzCUj3+ebjTG3e70e5JcxZqsxZsPI932b1+uBu4wxPzHG7DHGvOa4dpwx5lljzJsjHxu8XGOhISCXptckXSnpj86LxphTJF0j6X2SLpH0Q2NMMP/LQ55911o7d+THb7xeDNw38n19j6S/lXSKpGtHvv9RWhaOfN8z5qv4PaDhv9edbpf0O2vtiZJ+N/I5RhCQS5C1dqO1dlOMhz4u6RFrbb+19m1JmyUtyO/qAOTBAkmbrbVbrLUDkh7R8Pc/gCJkrf2jpANRlz8u6acjP/+ppMvzuaZCR0CG03RJ2x2f7xi5huL2RWPMn0f+C47/YisNfK/DSnrGGLPeGHOz14uBJ6ZaaztGft4paaqXiyk0ZV4vAO4wxjwnqTnGQ/9grf1lvtcD7yT6vSDp3yR9Q8N/WX5D0nck3Zi/1QHwyDnW2p3GmCmSnjXG/GWkyogSZK21xhjGmjkQkIuUtfaiDF62U9Lxjs9njFyDj6X6e8EYc5+kX7u8HBQGvtdLnLV258jHPcaYJzXcdkNALi27jTEt1toOY0yLpD1eL6iQ0GIBp6ckXWOMqTTGnCDpREnrPF4TXDTyh2LEFRrewIni94qkE40xJxhjKjS8Ofcpj9eEPDHGVBtjaiM/l3Sx+N4vRU9JumHk5zdI4n+XHagglyBjzBWSfiCpSdJKY0y7tfYj1trXjTGPSXpD0pCkL1hrQ16uFa672xgzV8MtFlsl/b2nq0FeWGuHjDFflPRbSUFJP7HWvu7xspA/UyU9aYyRhnPAw9bap71dEtxkjPm5pPMlNRpjdkj6mqQ7JT1mjLlJ0jZJn/RuhYWHk/QAAAAAB1osAAAAAAcCMgAAAOBAQAYAAAAcCMgAAACAAwEZAAAAcCAgAwAAAA4EZAAAAMCBgAwAAAA4/P+kMeNWiRiUgAAAAABJRU5ErkJggg==\n"},"metadata":{"needs_background":"light"}}]}]}